Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * A memslot-related performance benchmark.
4 *
5 * Copyright (C) 2021 Oracle and/or its affiliates.
6 *
7 * Basic guest setup / host vCPU thread code lifted from set_memory_region_test.
8 */
9#include <pthread.h>
10#include <sched.h>
11#include <semaphore.h>
12#include <stdatomic.h>
13#include <stdbool.h>
14#include <stdint.h>
15#include <stdio.h>
16#include <stdlib.h>
17#include <string.h>
18#include <sys/mman.h>
19#include <time.h>
20#include <unistd.h>
21
22#include <linux/compiler.h>
23#include <linux/sizes.h>
24
25#include <test_util.h>
26#include <kvm_util.h>
27#include <processor.h>
28
29#define MEM_EXTRA_SIZE SZ_64K
30
31#define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE)
32#define MEM_GPA SZ_256M
33#define MEM_AUX_GPA MEM_GPA
34#define MEM_SYNC_GPA MEM_AUX_GPA
35#define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE)
36#define MEM_TEST_SIZE (MEM_SIZE - MEM_EXTRA_SIZE)
37
38/*
39 * 32 MiB is max size that gets well over 100 iterations on 509 slots.
40 * Considering that each slot needs to have at least one page up to
41 * 8194 slots in use can then be tested (although with slightly
42 * limited resolution).
43 */
44#define MEM_SIZE_MAP (SZ_32M + MEM_EXTRA_SIZE)
45#define MEM_TEST_MAP_SIZE (MEM_SIZE_MAP - MEM_EXTRA_SIZE)
46
47/*
48 * 128 MiB is min size that fills 32k slots with at least one page in each
49 * while at the same time gets 100+ iterations in such test
50 *
51 * 2 MiB chunk size like a typical huge page
52 */
53#define MEM_TEST_UNMAP_SIZE SZ_128M
54#define MEM_TEST_UNMAP_CHUNK_SIZE SZ_2M
55
56/*
57 * For the move active test the middle of the test area is placed on
58 * a memslot boundary: half lies in the memslot being moved, half in
59 * other memslot(s).
60 *
61 * We have different number of memory slots, excluding the reserved
62 * memory slot 0, on various architectures and configurations. The
63 * memory size in this test is calculated by picking the maximal
64 * last memory slot's memory size, with alignment to the largest
65 * supported page size (64KB). In this way, the selected memory
66 * size for this test is compatible with test_memslot_move_prepare().
67 *
68 * architecture slots memory-per-slot memory-on-last-slot
69 * --------------------------------------------------------------
70 * x86-4KB 32763 16KB 160KB
71 * arm64-4KB 32766 16KB 112KB
72 * arm64-16KB 32766 16KB 112KB
73 * arm64-64KB 8192 64KB 128KB
74 */
75#define MEM_TEST_MOVE_SIZE (3 * SZ_64K)
76#define MEM_TEST_MOVE_GPA_DEST (MEM_GPA + MEM_SIZE)
77static_assert(MEM_TEST_MOVE_SIZE <= MEM_TEST_SIZE,
78 "invalid move test region size");
79
80#define MEM_TEST_VAL_1 0x1122334455667788
81#define MEM_TEST_VAL_2 0x99AABBCCDDEEFF00
82
83struct vm_data {
84 struct kvm_vm *vm;
85 struct kvm_vcpu *vcpu;
86 pthread_t vcpu_thread;
87 uint32_t nslots;
88 uint64_t npages;
89 uint64_t pages_per_slot;
90 void **hva_slots;
91 bool mmio_ok;
92 uint64_t mmio_gpa_min;
93 uint64_t mmio_gpa_max;
94};
95
96struct sync_area {
97 uint32_t guest_page_size;
98 atomic_bool start_flag;
99 atomic_bool exit_flag;
100 atomic_bool sync_flag;
101 void *move_area_ptr;
102};
103
104/*
105 * Technically, we need also for the atomic bool to be address-free, which
106 * is recommended, but not strictly required, by C11 for lockless
107 * implementations.
108 * However, in practice both GCC and Clang fulfill this requirement on
109 * all KVM-supported platforms.
110 */
111static_assert(ATOMIC_BOOL_LOCK_FREE == 2, "atomic bool is not lockless");
112
113static sem_t vcpu_ready;
114
115static bool map_unmap_verify;
116
117static bool verbose;
118#define pr_info_v(...) \
119 do { \
120 if (verbose) \
121 pr_info(__VA_ARGS__); \
122 } while (0)
123
124static void check_mmio_access(struct vm_data *data, struct kvm_run *run)
125{
126 TEST_ASSERT(data->mmio_ok, "Unexpected mmio exit");
127 TEST_ASSERT(run->mmio.is_write, "Unexpected mmio read");
128 TEST_ASSERT(run->mmio.len == 8,
129 "Unexpected exit mmio size = %u", run->mmio.len);
130 TEST_ASSERT(run->mmio.phys_addr >= data->mmio_gpa_min &&
131 run->mmio.phys_addr <= data->mmio_gpa_max,
132 "Unexpected exit mmio address = 0x%llx",
133 run->mmio.phys_addr);
134}
135
136static void *vcpu_worker(void *__data)
137{
138 struct vm_data *data = __data;
139 struct kvm_vcpu *vcpu = data->vcpu;
140 struct kvm_run *run = vcpu->run;
141 struct ucall uc;
142
143 while (1) {
144 vcpu_run(vcpu);
145
146 switch (get_ucall(vcpu, &uc)) {
147 case UCALL_SYNC:
148 TEST_ASSERT(uc.args[1] == 0,
149 "Unexpected sync ucall, got %lx",
150 (ulong)uc.args[1]);
151 sem_post(&vcpu_ready);
152 continue;
153 case UCALL_NONE:
154 if (run->exit_reason == KVM_EXIT_MMIO)
155 check_mmio_access(data, run);
156 else
157 goto done;
158 break;
159 case UCALL_ABORT:
160 REPORT_GUEST_ASSERT_1(uc, "val = %lu");
161 break;
162 case UCALL_DONE:
163 goto done;
164 default:
165 TEST_FAIL("Unknown ucall %lu", uc.cmd);
166 }
167 }
168
169done:
170 return NULL;
171}
172
173static void wait_for_vcpu(void)
174{
175 struct timespec ts;
176
177 TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),
178 "clock_gettime() failed: %d\n", errno);
179
180 ts.tv_sec += 2;
181 TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),
182 "sem_timedwait() failed: %d\n", errno);
183}
184
185static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages)
186{
187 uint64_t gpage, pgoffs;
188 uint32_t slot, slotoffs;
189 void *base;
190 uint32_t guest_page_size = data->vm->page_size;
191
192 TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");
193 TEST_ASSERT(gpa < MEM_GPA + data->npages * guest_page_size,
194 "Too high gpa to translate");
195 gpa -= MEM_GPA;
196
197 gpage = gpa / guest_page_size;
198 pgoffs = gpa % guest_page_size;
199 slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);
200 slotoffs = gpage - (slot * data->pages_per_slot);
201
202 if (rempages) {
203 uint64_t slotpages;
204
205 if (slot == data->nslots - 1)
206 slotpages = data->npages - slot * data->pages_per_slot;
207 else
208 slotpages = data->pages_per_slot;
209
210 TEST_ASSERT(!pgoffs,
211 "Asking for remaining pages in slot but gpa not page aligned");
212 *rempages = slotpages - slotoffs;
213 }
214
215 base = data->hva_slots[slot];
216 return (uint8_t *)base + slotoffs * guest_page_size + pgoffs;
217}
218
219static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)
220{
221 uint32_t guest_page_size = data->vm->page_size;
222
223 TEST_ASSERT(slot < data->nslots, "Too high slot number");
224
225 return MEM_GPA + slot * data->pages_per_slot * guest_page_size;
226}
227
228static struct vm_data *alloc_vm(void)
229{
230 struct vm_data *data;
231
232 data = malloc(sizeof(*data));
233 TEST_ASSERT(data, "malloc(vmdata) failed");
234
235 data->vm = NULL;
236 data->vcpu = NULL;
237 data->hva_slots = NULL;
238
239 return data;
240}
241
242static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
243 uint64_t pages_per_slot, uint64_t rempages)
244{
245 if (!pages_per_slot)
246 return false;
247
248 if ((pages_per_slot * guest_page_size) % host_page_size)
249 return false;
250
251 if ((rempages * guest_page_size) % host_page_size)
252 return false;
253
254 return true;
255}
256
257
258static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
259{
260 uint32_t guest_page_size = data->vm->page_size;
261 uint64_t mempages, pages_per_slot, rempages;
262 uint64_t slots;
263
264 mempages = data->npages;
265 slots = data->nslots;
266 while (--slots > 1) {
267 pages_per_slot = mempages / slots;
268 if (!pages_per_slot)
269 continue;
270
271 rempages = mempages % pages_per_slot;
272 if (check_slot_pages(host_page_size, guest_page_size,
273 pages_per_slot, rempages))
274 return slots + 1; /* slot 0 is reserved */
275 }
276
277 return 0;
278}
279
280static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
281 void *guest_code, uint64_t mem_size,
282 struct timespec *slot_runtime)
283{
284 uint64_t mempages, rempages;
285 uint64_t guest_addr;
286 uint32_t slot, host_page_size, guest_page_size;
287 struct timespec tstart;
288 struct sync_area *sync;
289
290 host_page_size = getpagesize();
291 guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
292 mempages = mem_size / guest_page_size;
293
294 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
295 TEST_ASSERT(data->vm->page_size == guest_page_size, "Invalid VM page size");
296
297 data->npages = mempages;
298 TEST_ASSERT(data->npages > 1, "Can't test without any memory");
299 data->nslots = nslots;
300 data->pages_per_slot = data->npages / data->nslots;
301 rempages = data->npages % data->nslots;
302 if (!check_slot_pages(host_page_size, guest_page_size,
303 data->pages_per_slot, rempages)) {
304 *maxslots = get_max_slots(data, host_page_size);
305 return false;
306 }
307
308 data->hva_slots = malloc(sizeof(*data->hva_slots) * data->nslots);
309 TEST_ASSERT(data->hva_slots, "malloc() fail");
310
311 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
312
313 pr_info_v("Adding slots 1..%i, each slot with %"PRIu64" pages + %"PRIu64" extra pages last\n",
314 data->nslots, data->pages_per_slot, rempages);
315
316 clock_gettime(CLOCK_MONOTONIC, &tstart);
317 for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
318 uint64_t npages;
319
320 npages = data->pages_per_slot;
321 if (slot == data->nslots)
322 npages += rempages;
323
324 vm_userspace_mem_region_add(data->vm, VM_MEM_SRC_ANONYMOUS,
325 guest_addr, slot, npages,
326 0);
327 guest_addr += npages * guest_page_size;
328 }
329 *slot_runtime = timespec_elapsed(tstart);
330
331 for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
332 uint64_t npages;
333 uint64_t gpa;
334
335 npages = data->pages_per_slot;
336 if (slot == data->nslots)
337 npages += rempages;
338
339 gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot);
340 TEST_ASSERT(gpa == guest_addr,
341 "vm_phy_pages_alloc() failed\n");
342
343 data->hva_slots[slot - 1] = addr_gpa2hva(data->vm, guest_addr);
344 memset(data->hva_slots[slot - 1], 0, npages * guest_page_size);
345
346 guest_addr += npages * guest_page_size;
347 }
348
349 virt_map(data->vm, MEM_GPA, MEM_GPA, data->npages);
350
351 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
352 atomic_init(&sync->start_flag, false);
353 atomic_init(&sync->exit_flag, false);
354 atomic_init(&sync->sync_flag, false);
355
356 data->mmio_ok = false;
357
358 return true;
359}
360
361static void launch_vm(struct vm_data *data)
362{
363 pr_info_v("Launching the test VM\n");
364
365 pthread_create(&data->vcpu_thread, NULL, vcpu_worker, data);
366
367 /* Ensure the guest thread is spun up. */
368 wait_for_vcpu();
369}
370
371static void free_vm(struct vm_data *data)
372{
373 kvm_vm_free(data->vm);
374 free(data->hva_slots);
375 free(data);
376}
377
378static void wait_guest_exit(struct vm_data *data)
379{
380 pthread_join(data->vcpu_thread, NULL);
381}
382
383static void let_guest_run(struct sync_area *sync)
384{
385 atomic_store_explicit(&sync->start_flag, true, memory_order_release);
386}
387
388static void guest_spin_until_start(void)
389{
390 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
391
392 while (!atomic_load_explicit(&sync->start_flag, memory_order_acquire))
393 ;
394}
395
396static void make_guest_exit(struct sync_area *sync)
397{
398 atomic_store_explicit(&sync->exit_flag, true, memory_order_release);
399}
400
401static bool _guest_should_exit(void)
402{
403 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
404
405 return atomic_load_explicit(&sync->exit_flag, memory_order_acquire);
406}
407
408#define guest_should_exit() unlikely(_guest_should_exit())
409
410/*
411 * noinline so we can easily see how much time the host spends waiting
412 * for the guest.
413 * For the same reason use alarm() instead of polling clock_gettime()
414 * to implement a wait timeout.
415 */
416static noinline void host_perform_sync(struct sync_area *sync)
417{
418 alarm(2);
419
420 atomic_store_explicit(&sync->sync_flag, true, memory_order_release);
421 while (atomic_load_explicit(&sync->sync_flag, memory_order_acquire))
422 ;
423
424 alarm(0);
425}
426
427static bool guest_perform_sync(void)
428{
429 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
430 bool expected;
431
432 do {
433 if (guest_should_exit())
434 return false;
435
436 expected = true;
437 } while (!atomic_compare_exchange_weak_explicit(&sync->sync_flag,
438 &expected, false,
439 memory_order_acq_rel,
440 memory_order_relaxed));
441
442 return true;
443}
444
445static void guest_code_test_memslot_move(void)
446{
447 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
448 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
449 uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);
450
451 GUEST_SYNC(0);
452
453 guest_spin_until_start();
454
455 while (!guest_should_exit()) {
456 uintptr_t ptr;
457
458 for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;
459 ptr += page_size)
460 *(uint64_t *)ptr = MEM_TEST_VAL_1;
461
462 /*
463 * No host sync here since the MMIO exits are so expensive
464 * that the host would spend most of its time waiting for
465 * the guest and so instead of measuring memslot move
466 * performance we would measure the performance and
467 * likelihood of MMIO exits
468 */
469 }
470
471 GUEST_DONE();
472}
473
474static void guest_code_test_memslot_map(void)
475{
476 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
477 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
478
479 GUEST_SYNC(0);
480
481 guest_spin_until_start();
482
483 while (1) {
484 uintptr_t ptr;
485
486 for (ptr = MEM_TEST_GPA;
487 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
488 ptr += page_size)
489 *(uint64_t *)ptr = MEM_TEST_VAL_1;
490
491 if (!guest_perform_sync())
492 break;
493
494 for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
495 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE;
496 ptr += page_size)
497 *(uint64_t *)ptr = MEM_TEST_VAL_2;
498
499 if (!guest_perform_sync())
500 break;
501 }
502
503 GUEST_DONE();
504}
505
506static void guest_code_test_memslot_unmap(void)
507{
508 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
509
510 GUEST_SYNC(0);
511
512 guest_spin_until_start();
513
514 while (1) {
515 uintptr_t ptr = MEM_TEST_GPA;
516
517 /*
518 * We can afford to access (map) just a small number of pages
519 * per host sync as otherwise the host will spend
520 * a significant amount of its time waiting for the guest
521 * (instead of doing unmap operations), so this will
522 * effectively turn this test into a map performance test.
523 *
524 * Just access a single page to be on the safe side.
525 */
526 *(uint64_t *)ptr = MEM_TEST_VAL_1;
527
528 if (!guest_perform_sync())
529 break;
530
531 ptr += MEM_TEST_UNMAP_SIZE / 2;
532 *(uint64_t *)ptr = MEM_TEST_VAL_2;
533
534 if (!guest_perform_sync())
535 break;
536 }
537
538 GUEST_DONE();
539}
540
541static void guest_code_test_memslot_rw(void)
542{
543 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
544 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
545
546 GUEST_SYNC(0);
547
548 guest_spin_until_start();
549
550 while (1) {
551 uintptr_t ptr;
552
553 for (ptr = MEM_TEST_GPA;
554 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size)
555 *(uint64_t *)ptr = MEM_TEST_VAL_1;
556
557 if (!guest_perform_sync())
558 break;
559
560 for (ptr = MEM_TEST_GPA + page_size / 2;
561 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size) {
562 uint64_t val = *(uint64_t *)ptr;
563
564 GUEST_ASSERT_1(val == MEM_TEST_VAL_2, val);
565 *(uint64_t *)ptr = 0;
566 }
567
568 if (!guest_perform_sync())
569 break;
570 }
571
572 GUEST_DONE();
573}
574
575static bool test_memslot_move_prepare(struct vm_data *data,
576 struct sync_area *sync,
577 uint64_t *maxslots, bool isactive)
578{
579 uint32_t guest_page_size = data->vm->page_size;
580 uint64_t movesrcgpa, movetestgpa;
581
582 movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
583
584 if (isactive) {
585 uint64_t lastpages;
586
587 vm_gpa2hva(data, movesrcgpa, &lastpages);
588 if (lastpages * guest_page_size < MEM_TEST_MOVE_SIZE / 2) {
589 *maxslots = 0;
590 return false;
591 }
592 }
593
594 movetestgpa = movesrcgpa - (MEM_TEST_MOVE_SIZE / (isactive ? 2 : 1));
595 sync->move_area_ptr = (void *)movetestgpa;
596
597 if (isactive) {
598 data->mmio_ok = true;
599 data->mmio_gpa_min = movesrcgpa;
600 data->mmio_gpa_max = movesrcgpa + MEM_TEST_MOVE_SIZE / 2 - 1;
601 }
602
603 return true;
604}
605
606static bool test_memslot_move_prepare_active(struct vm_data *data,
607 struct sync_area *sync,
608 uint64_t *maxslots)
609{
610 return test_memslot_move_prepare(data, sync, maxslots, true);
611}
612
613static bool test_memslot_move_prepare_inactive(struct vm_data *data,
614 struct sync_area *sync,
615 uint64_t *maxslots)
616{
617 return test_memslot_move_prepare(data, sync, maxslots, false);
618}
619
620static void test_memslot_move_loop(struct vm_data *data, struct sync_area *sync)
621{
622 uint64_t movesrcgpa;
623
624 movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
625 vm_mem_region_move(data->vm, data->nslots - 1 + 1,
626 MEM_TEST_MOVE_GPA_DEST);
627 vm_mem_region_move(data->vm, data->nslots - 1 + 1, movesrcgpa);
628}
629
630static void test_memslot_do_unmap(struct vm_data *data,
631 uint64_t offsp, uint64_t count)
632{
633 uint64_t gpa, ctr;
634 uint32_t guest_page_size = data->vm->page_size;
635
636 for (gpa = MEM_TEST_GPA + offsp * guest_page_size, ctr = 0; ctr < count; ) {
637 uint64_t npages;
638 void *hva;
639 int ret;
640
641 hva = vm_gpa2hva(data, gpa, &npages);
642 TEST_ASSERT(npages, "Empty memory slot at gptr 0x%"PRIx64, gpa);
643 npages = min(npages, count - ctr);
644 ret = madvise(hva, npages * guest_page_size, MADV_DONTNEED);
645 TEST_ASSERT(!ret,
646 "madvise(%p, MADV_DONTNEED) on VM memory should not fail for gptr 0x%"PRIx64,
647 hva, gpa);
648 ctr += npages;
649 gpa += npages * guest_page_size;
650 }
651 TEST_ASSERT(ctr == count,
652 "madvise(MADV_DONTNEED) should exactly cover all of the requested area");
653}
654
655static void test_memslot_map_unmap_check(struct vm_data *data,
656 uint64_t offsp, uint64_t valexp)
657{
658 uint64_t gpa;
659 uint64_t *val;
660 uint32_t guest_page_size = data->vm->page_size;
661
662 if (!map_unmap_verify)
663 return;
664
665 gpa = MEM_TEST_GPA + offsp * guest_page_size;
666 val = (typeof(val))vm_gpa2hva(data, gpa, NULL);
667 TEST_ASSERT(*val == valexp,
668 "Guest written values should read back correctly before unmap (%"PRIu64" vs %"PRIu64" @ %"PRIx64")",
669 *val, valexp, gpa);
670 *val = 0;
671}
672
673static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
674{
675 uint32_t guest_page_size = data->vm->page_size;
676 uint64_t guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
677
678 /*
679 * Unmap the second half of the test area while guest writes to (maps)
680 * the first half.
681 */
682 test_memslot_do_unmap(data, guest_pages / 2, guest_pages / 2);
683
684 /*
685 * Wait for the guest to finish writing the first half of the test
686 * area, verify the written value on the first and the last page of
687 * this area and then unmap it.
688 * Meanwhile, the guest is writing to (mapping) the second half of
689 * the test area.
690 */
691 host_perform_sync(sync);
692 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
693 test_memslot_map_unmap_check(data, guest_pages / 2 - 1, MEM_TEST_VAL_1);
694 test_memslot_do_unmap(data, 0, guest_pages / 2);
695
696
697 /*
698 * Wait for the guest to finish writing the second half of the test
699 * area and verify the written value on the first and the last page
700 * of this area.
701 * The area will be unmapped at the beginning of the next loop
702 * iteration.
703 * Meanwhile, the guest is writing to (mapping) the first half of
704 * the test area.
705 */
706 host_perform_sync(sync);
707 test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
708 test_memslot_map_unmap_check(data, guest_pages - 1, MEM_TEST_VAL_2);
709}
710
711static void test_memslot_unmap_loop_common(struct vm_data *data,
712 struct sync_area *sync,
713 uint64_t chunk)
714{
715 uint32_t guest_page_size = data->vm->page_size;
716 uint64_t guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
717 uint64_t ctr;
718
719 /*
720 * Wait for the guest to finish mapping page(s) in the first half
721 * of the test area, verify the written value and then perform unmap
722 * of this area.
723 * Meanwhile, the guest is writing to (mapping) page(s) in the second
724 * half of the test area.
725 */
726 host_perform_sync(sync);
727 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
728 for (ctr = 0; ctr < guest_pages / 2; ctr += chunk)
729 test_memslot_do_unmap(data, ctr, chunk);
730
731 /* Likewise, but for the opposite host / guest areas */
732 host_perform_sync(sync);
733 test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
734 for (ctr = guest_pages / 2; ctr < guest_pages; ctr += chunk)
735 test_memslot_do_unmap(data, ctr, chunk);
736}
737
738static void test_memslot_unmap_loop(struct vm_data *data,
739 struct sync_area *sync)
740{
741 uint32_t host_page_size = getpagesize();
742 uint32_t guest_page_size = data->vm->page_size;
743 uint64_t guest_chunk_pages = guest_page_size >= host_page_size ?
744 1 : host_page_size / guest_page_size;
745
746 test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
747}
748
749static void test_memslot_unmap_loop_chunked(struct vm_data *data,
750 struct sync_area *sync)
751{
752 uint32_t guest_page_size = data->vm->page_size;
753 uint64_t guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
754
755 test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
756}
757
758static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
759{
760 uint64_t gptr;
761 uint32_t guest_page_size = data->vm->page_size;
762
763 for (gptr = MEM_TEST_GPA + guest_page_size / 2;
764 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size)
765 *(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
766
767 host_perform_sync(sync);
768
769 for (gptr = MEM_TEST_GPA;
770 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size) {
771 uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
772 uint64_t val = *vptr;
773
774 TEST_ASSERT(val == MEM_TEST_VAL_1,
775 "Guest written values should read back correctly (is %"PRIu64" @ %"PRIx64")",
776 val, gptr);
777 *vptr = 0;
778 }
779
780 host_perform_sync(sync);
781}
782
783struct test_data {
784 const char *name;
785 uint64_t mem_size;
786 void (*guest_code)(void);
787 bool (*prepare)(struct vm_data *data, struct sync_area *sync,
788 uint64_t *maxslots);
789 void (*loop)(struct vm_data *data, struct sync_area *sync);
790};
791
792static bool test_execute(int nslots, uint64_t *maxslots,
793 unsigned int maxtime,
794 const struct test_data *tdata,
795 uint64_t *nloops,
796 struct timespec *slot_runtime,
797 struct timespec *guest_runtime)
798{
799 uint64_t mem_size = tdata->mem_size ? : MEM_SIZE;
800 struct vm_data *data;
801 struct sync_area *sync;
802 struct timespec tstart;
803 bool ret = true;
804
805 data = alloc_vm();
806 if (!prepare_vm(data, nslots, maxslots, tdata->guest_code,
807 mem_size, slot_runtime)) {
808 ret = false;
809 goto exit_free;
810 }
811
812 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
813
814 sync->guest_page_size = data->vm->page_size;
815 if (tdata->prepare &&
816 !tdata->prepare(data, sync, maxslots)) {
817 ret = false;
818 goto exit_free;
819 }
820
821 launch_vm(data);
822
823 clock_gettime(CLOCK_MONOTONIC, &tstart);
824 let_guest_run(sync);
825
826 while (1) {
827 *guest_runtime = timespec_elapsed(tstart);
828 if (guest_runtime->tv_sec >= maxtime)
829 break;
830
831 tdata->loop(data, sync);
832
833 (*nloops)++;
834 }
835
836 make_guest_exit(sync);
837 wait_guest_exit(data);
838
839exit_free:
840 free_vm(data);
841
842 return ret;
843}
844
845static const struct test_data tests[] = {
846 {
847 .name = "map",
848 .mem_size = MEM_SIZE_MAP,
849 .guest_code = guest_code_test_memslot_map,
850 .loop = test_memslot_map_loop,
851 },
852 {
853 .name = "unmap",
854 .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
855 .guest_code = guest_code_test_memslot_unmap,
856 .loop = test_memslot_unmap_loop,
857 },
858 {
859 .name = "unmap chunked",
860 .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
861 .guest_code = guest_code_test_memslot_unmap,
862 .loop = test_memslot_unmap_loop_chunked,
863 },
864 {
865 .name = "move active area",
866 .guest_code = guest_code_test_memslot_move,
867 .prepare = test_memslot_move_prepare_active,
868 .loop = test_memslot_move_loop,
869 },
870 {
871 .name = "move inactive area",
872 .guest_code = guest_code_test_memslot_move,
873 .prepare = test_memslot_move_prepare_inactive,
874 .loop = test_memslot_move_loop,
875 },
876 {
877 .name = "RW",
878 .guest_code = guest_code_test_memslot_rw,
879 .loop = test_memslot_rw_loop
880 },
881};
882
883#define NTESTS ARRAY_SIZE(tests)
884
885struct test_args {
886 int tfirst;
887 int tlast;
888 int nslots;
889 int seconds;
890 int runs;
891};
892
893static void help(char *name, struct test_args *targs)
894{
895 int ctr;
896
897 pr_info("usage: %s [-h] [-v] [-d] [-s slots] [-f first_test] [-e last_test] [-l test_length] [-r run_count]\n",
898 name);
899 pr_info(" -h: print this help screen.\n");
900 pr_info(" -v: enable verbose mode (not for benchmarking).\n");
901 pr_info(" -d: enable extra debug checks.\n");
902 pr_info(" -s: specify memslot count cap (-1 means no cap; currently: %i)\n",
903 targs->nslots);
904 pr_info(" -f: specify the first test to run (currently: %i; max %zu)\n",
905 targs->tfirst, NTESTS - 1);
906 pr_info(" -e: specify the last test to run (currently: %i; max %zu)\n",
907 targs->tlast, NTESTS - 1);
908 pr_info(" -l: specify the test length in seconds (currently: %i)\n",
909 targs->seconds);
910 pr_info(" -r: specify the number of runs per test (currently: %i)\n",
911 targs->runs);
912
913 pr_info("\nAvailable tests:\n");
914 for (ctr = 0; ctr < NTESTS; ctr++)
915 pr_info("%d: %s\n", ctr, tests[ctr].name);
916}
917
918static bool check_memory_sizes(void)
919{
920 uint32_t host_page_size = getpagesize();
921 uint32_t guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
922
923 if (host_page_size > SZ_64K || guest_page_size > SZ_64K) {
924 pr_info("Unsupported page size on host (0x%x) or guest (0x%x)\n",
925 host_page_size, guest_page_size);
926 return false;
927 }
928
929 if (MEM_SIZE % guest_page_size ||
930 MEM_TEST_SIZE % guest_page_size) {
931 pr_info("invalid MEM_SIZE or MEM_TEST_SIZE\n");
932 return false;
933 }
934
935 if (MEM_SIZE_MAP % guest_page_size ||
936 MEM_TEST_MAP_SIZE % guest_page_size ||
937 (MEM_TEST_MAP_SIZE / guest_page_size) <= 2 ||
938 (MEM_TEST_MAP_SIZE / guest_page_size) % 2) {
939 pr_info("invalid MEM_SIZE_MAP or MEM_TEST_MAP_SIZE\n");
940 return false;
941 }
942
943 if (MEM_TEST_UNMAP_SIZE > MEM_TEST_SIZE ||
944 MEM_TEST_UNMAP_SIZE % guest_page_size ||
945 (MEM_TEST_UNMAP_SIZE / guest_page_size) %
946 (2 * MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size)) {
947 pr_info("invalid MEM_TEST_UNMAP_SIZE or MEM_TEST_UNMAP_CHUNK_SIZE\n");
948 return false;
949 }
950
951 return true;
952}
953
954static bool parse_args(int argc, char *argv[],
955 struct test_args *targs)
956{
957 uint32_t max_mem_slots;
958 int opt;
959
960 while ((opt = getopt(argc, argv, "hvds:f:e:l:r:")) != -1) {
961 switch (opt) {
962 case 'h':
963 default:
964 help(argv[0], targs);
965 return false;
966 case 'v':
967 verbose = true;
968 break;
969 case 'd':
970 map_unmap_verify = true;
971 break;
972 case 's':
973 targs->nslots = atoi_paranoid(optarg);
974 if (targs->nslots <= 1 && targs->nslots != -1) {
975 pr_info("Slot count cap must be larger than 1 or -1 for no cap\n");
976 return false;
977 }
978 break;
979 case 'f':
980 targs->tfirst = atoi_non_negative("First test", optarg);
981 break;
982 case 'e':
983 targs->tlast = atoi_non_negative("Last test", optarg);
984 if (targs->tlast >= NTESTS) {
985 pr_info("Last test to run has to be non-negative and less than %zu\n",
986 NTESTS);
987 return false;
988 }
989 break;
990 case 'l':
991 targs->seconds = atoi_non_negative("Test length", optarg);
992 break;
993 case 'r':
994 targs->runs = atoi_positive("Runs per test", optarg);
995 break;
996 }
997 }
998
999 if (optind < argc) {
1000 help(argv[0], targs);
1001 return false;
1002 }
1003
1004 if (targs->tfirst > targs->tlast) {
1005 pr_info("First test to run cannot be greater than the last test to run\n");
1006 return false;
1007 }
1008
1009 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
1010 if (max_mem_slots <= 1) {
1011 pr_info("KVM_CAP_NR_MEMSLOTS should be greater than 1\n");
1012 return false;
1013 }
1014
1015 /* Memory slot 0 is reserved */
1016 if (targs->nslots == -1)
1017 targs->nslots = max_mem_slots - 1;
1018 else
1019 targs->nslots = min_t(int, targs->nslots, max_mem_slots) - 1;
1020
1021 pr_info_v("Allowed Number of memory slots: %"PRIu32"\n",
1022 targs->nslots + 1);
1023
1024 return true;
1025}
1026
1027struct test_result {
1028 struct timespec slot_runtime, guest_runtime, iter_runtime;
1029 int64_t slottimens, runtimens;
1030 uint64_t nloops;
1031};
1032
1033static bool test_loop(const struct test_data *data,
1034 const struct test_args *targs,
1035 struct test_result *rbestslottime,
1036 struct test_result *rbestruntime)
1037{
1038 uint64_t maxslots;
1039 struct test_result result;
1040
1041 result.nloops = 0;
1042 if (!test_execute(targs->nslots, &maxslots, targs->seconds, data,
1043 &result.nloops,
1044 &result.slot_runtime, &result.guest_runtime)) {
1045 if (maxslots)
1046 pr_info("Memslot count too high for this test, decrease the cap (max is %"PRIu64")\n",
1047 maxslots);
1048 else
1049 pr_info("Memslot count may be too high for this test, try adjusting the cap\n");
1050
1051 return false;
1052 }
1053
1054 pr_info("Test took %ld.%.9lds for slot setup + %ld.%.9lds all iterations\n",
1055 result.slot_runtime.tv_sec, result.slot_runtime.tv_nsec,
1056 result.guest_runtime.tv_sec, result.guest_runtime.tv_nsec);
1057 if (!result.nloops) {
1058 pr_info("No full loops done - too short test time or system too loaded?\n");
1059 return true;
1060 }
1061
1062 result.iter_runtime = timespec_div(result.guest_runtime,
1063 result.nloops);
1064 pr_info("Done %"PRIu64" iterations, avg %ld.%.9lds each\n",
1065 result.nloops,
1066 result.iter_runtime.tv_sec,
1067 result.iter_runtime.tv_nsec);
1068 result.slottimens = timespec_to_ns(result.slot_runtime);
1069 result.runtimens = timespec_to_ns(result.iter_runtime);
1070
1071 /*
1072 * Only rank the slot setup time for tests using the whole test memory
1073 * area so they are comparable
1074 */
1075 if (!data->mem_size &&
1076 (!rbestslottime->slottimens ||
1077 result.slottimens < rbestslottime->slottimens))
1078 *rbestslottime = result;
1079 if (!rbestruntime->runtimens ||
1080 result.runtimens < rbestruntime->runtimens)
1081 *rbestruntime = result;
1082
1083 return true;
1084}
1085
1086int main(int argc, char *argv[])
1087{
1088 struct test_args targs = {
1089 .tfirst = 0,
1090 .tlast = NTESTS - 1,
1091 .nslots = -1,
1092 .seconds = 5,
1093 .runs = 1,
1094 };
1095 struct test_result rbestslottime;
1096 int tctr;
1097
1098 if (!check_memory_sizes())
1099 return -1;
1100
1101 if (!parse_args(argc, argv, &targs))
1102 return -1;
1103
1104 rbestslottime.slottimens = 0;
1105 for (tctr = targs.tfirst; tctr <= targs.tlast; tctr++) {
1106 const struct test_data *data = &tests[tctr];
1107 unsigned int runctr;
1108 struct test_result rbestruntime;
1109
1110 if (tctr > targs.tfirst)
1111 pr_info("\n");
1112
1113 pr_info("Testing %s performance with %i runs, %d seconds each\n",
1114 data->name, targs.runs, targs.seconds);
1115
1116 rbestruntime.runtimens = 0;
1117 for (runctr = 0; runctr < targs.runs; runctr++)
1118 if (!test_loop(data, &targs,
1119 &rbestslottime, &rbestruntime))
1120 break;
1121
1122 if (rbestruntime.runtimens)
1123 pr_info("Best runtime result was %ld.%.9lds per iteration (with %"PRIu64" iterations)\n",
1124 rbestruntime.iter_runtime.tv_sec,
1125 rbestruntime.iter_runtime.tv_nsec,
1126 rbestruntime.nloops);
1127 }
1128
1129 if (rbestslottime.slottimens)
1130 pr_info("Best slot setup time for the whole test area was %ld.%.9lds\n",
1131 rbestslottime.slot_runtime.tv_sec,
1132 rbestslottime.slot_runtime.tv_nsec);
1133
1134 return 0;
1135}