Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8#include "uffd-common.h"
9
10#include "../../../../mm/gup_test.h"
11
12#ifdef __NR_userfaultfd
13
14/* The unit test doesn't need a large or random size, make it 32MB for now */
15#define UFFD_TEST_MEM_SIZE (32UL << 20)
16
17#define MEM_ANON BIT_ULL(0)
18#define MEM_SHMEM BIT_ULL(1)
19#define MEM_SHMEM_PRIVATE BIT_ULL(2)
20#define MEM_HUGETLB BIT_ULL(3)
21#define MEM_HUGETLB_PRIVATE BIT_ULL(4)
22
23#define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
24 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
25
26#define ALIGN_UP(x, align_to) \
27 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
28
29struct mem_type {
30 const char *name;
31 unsigned int mem_flag;
32 uffd_test_ops_t *mem_ops;
33 bool shared;
34};
35typedef struct mem_type mem_type_t;
36
37mem_type_t mem_types[] = {
38 {
39 .name = "anon",
40 .mem_flag = MEM_ANON,
41 .mem_ops = &anon_uffd_test_ops,
42 .shared = false,
43 },
44 {
45 .name = "shmem",
46 .mem_flag = MEM_SHMEM,
47 .mem_ops = &shmem_uffd_test_ops,
48 .shared = true,
49 },
50 {
51 .name = "shmem-private",
52 .mem_flag = MEM_SHMEM_PRIVATE,
53 .mem_ops = &shmem_uffd_test_ops,
54 .shared = false,
55 },
56 {
57 .name = "hugetlb",
58 .mem_flag = MEM_HUGETLB,
59 .mem_ops = &hugetlb_uffd_test_ops,
60 .shared = true,
61 },
62 {
63 .name = "hugetlb-private",
64 .mem_flag = MEM_HUGETLB_PRIVATE,
65 .mem_ops = &hugetlb_uffd_test_ops,
66 .shared = false,
67 },
68};
69
70/* Arguments to be passed over to each uffd unit test */
71struct uffd_test_args {
72 mem_type_t *mem_type;
73};
74typedef struct uffd_test_args uffd_test_args_t;
75
76/* Returns: UFFD_TEST_* */
77typedef void (*uffd_test_fn)(uffd_test_args_t *);
78
79typedef struct {
80 const char *name;
81 uffd_test_fn uffd_fn;
82 unsigned int mem_targets;
83 uint64_t uffd_feature_required;
84 uffd_test_case_ops_t *test_case_ops;
85} uffd_test_case_t;
86
87static void uffd_test_report(void)
88{
89 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
90 ksft_get_pass_cnt(),
91 ksft_get_xskip_cnt(),
92 ksft_get_fail_cnt(),
93 ksft_test_num());
94}
95
96static void uffd_test_pass(void)
97{
98 printf("done\n");
99 ksft_inc_pass_cnt();
100}
101
102#define uffd_test_start(...) do { \
103 printf("Testing "); \
104 printf(__VA_ARGS__); \
105 printf("... "); \
106 fflush(stdout); \
107 } while (0)
108
109#define uffd_test_fail(...) do { \
110 printf("failed [reason: "); \
111 printf(__VA_ARGS__); \
112 printf("]\n"); \
113 ksft_inc_fail_cnt(); \
114 } while (0)
115
116static void uffd_test_skip(const char *message)
117{
118 printf("skipped [reason: %s]\n", message);
119 ksft_inc_xskip_cnt();
120}
121
122/*
123 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll
124 * return 1 even if some test failed as long as uffd supported, because in
125 * that case we still want to proceed with the rest uffd unit tests.
126 */
127static int test_uffd_api(bool use_dev)
128{
129 struct uffdio_api uffdio_api;
130 int uffd;
131
132 uffd_test_start("UFFDIO_API (with %s)",
133 use_dev ? "/dev/userfaultfd" : "syscall");
134
135 if (use_dev)
136 uffd = uffd_open_dev(UFFD_FLAGS);
137 else
138 uffd = uffd_open_sys(UFFD_FLAGS);
139 if (uffd < 0) {
140 uffd_test_skip("cannot open userfaultfd handle");
141 return 0;
142 }
143
144 /* Test wrong UFFD_API */
145 uffdio_api.api = 0xab;
146 uffdio_api.features = 0;
147 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
148 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
149 goto out;
150 }
151
152 /* Test wrong feature bit */
153 uffdio_api.api = UFFD_API;
154 uffdio_api.features = BIT_ULL(63);
155 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
156 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
157 goto out;
158 }
159
160 /* Test normal UFFDIO_API */
161 uffdio_api.api = UFFD_API;
162 uffdio_api.features = 0;
163 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
164 uffd_test_fail("UFFDIO_API should succeed but failed");
165 goto out;
166 }
167
168 /* Test double requests of UFFDIO_API with a random feature set */
169 uffdio_api.features = BIT_ULL(0);
170 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
171 uffd_test_fail("UFFDIO_API should reject initialized uffd");
172 goto out;
173 }
174
175 uffd_test_pass();
176out:
177 close(uffd);
178 /* We have a valid uffd handle */
179 return 1;
180}
181
182/*
183 * This function initializes the global variables. TODO: remove global
184 * vars and then remove this.
185 */
186static int
187uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
188 mem_type_t *mem_type, const char **errmsg)
189{
190 map_shared = mem_type->shared;
191 uffd_test_ops = mem_type->mem_ops;
192 uffd_test_case_ops = test->test_case_ops;
193
194 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
195 page_size = default_huge_page_size();
196 else
197 page_size = psize();
198
199 nr_pages = UFFD_TEST_MEM_SIZE / page_size;
200 /* TODO: remove this global var.. it's so ugly */
201 nr_cpus = 1;
202
203 /* Initialize test arguments */
204 args->mem_type = mem_type;
205
206 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
207}
208
209static bool uffd_feature_supported(uffd_test_case_t *test)
210{
211 uint64_t features;
212
213 if (uffd_get_features(&features))
214 return false;
215
216 return (features & test->uffd_feature_required) ==
217 test->uffd_feature_required;
218}
219
220static int pagemap_open(void)
221{
222 int fd = open("/proc/self/pagemap", O_RDONLY);
223
224 if (fd < 0)
225 err("open pagemap");
226
227 return fd;
228}
229
230/* This macro let __LINE__ works in err() */
231#define pagemap_check_wp(value, wp) do { \
232 if (!!(value & PM_UFFD_WP) != wp) \
233 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
234 } while (0)
235
236typedef struct {
237 int parent_uffd, child_uffd;
238} fork_event_args;
239
240static void *fork_event_consumer(void *data)
241{
242 fork_event_args *args = data;
243 struct uffd_msg msg = { 0 };
244
245 /* Read until a full msg received */
246 while (uffd_read_msg(args->parent_uffd, &msg));
247
248 if (msg.event != UFFD_EVENT_FORK)
249 err("wrong message: %u\n", msg.event);
250
251 /* Just to be properly freed later */
252 args->child_uffd = msg.arg.fork.ufd;
253 return NULL;
254}
255
256typedef struct {
257 int gup_fd;
258 bool pinned;
259} pin_args;
260
261/*
262 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
263 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
264 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
265 * CoW, etc.).
266 */
267static int pin_pages(pin_args *args, void *buffer, size_t size)
268{
269 struct pin_longterm_test test = {
270 .addr = (uintptr_t)buffer,
271 .size = size,
272 /* Read-only pins */
273 .flags = 0,
274 };
275
276 if (args->pinned)
277 err("already pinned");
278
279 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
280 if (args->gup_fd < 0)
281 return -errno;
282
283 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
284 /* Even if gup_test existed, can be an old gup_test / kernel */
285 close(args->gup_fd);
286 return -errno;
287 }
288 args->pinned = true;
289 return 0;
290}
291
292static void unpin_pages(pin_args *args)
293{
294 if (!args->pinned)
295 err("unpin without pin first");
296 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
297 err("PIN_LONGTERM_TEST_STOP");
298 close(args->gup_fd);
299 args->pinned = false;
300}
301
302static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
303{
304 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
305 pthread_t thread;
306 pid_t child;
307 uint64_t value;
308 int fd, result;
309
310 /* Prepare a thread to resolve EVENT_FORK */
311 if (with_event) {
312 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
313 err("pthread_create()");
314 }
315
316 child = fork();
317 if (!child) {
318 /* Open the pagemap fd of the child itself */
319 pin_args args = {};
320
321 fd = pagemap_open();
322
323 if (test_pin && pin_pages(&args, area_dst, page_size))
324 /*
325 * Normally when reach here we have pinned in
326 * previous tests, so shouldn't fail anymore
327 */
328 err("pin page failed in child");
329
330 value = pagemap_get_entry(fd, area_dst);
331 /*
332 * After fork(), we should handle uffd-wp bit differently:
333 *
334 * (1) when with EVENT_FORK, it should persist
335 * (2) when without EVENT_FORK, it should be dropped
336 */
337 pagemap_check_wp(value, with_event);
338 if (test_pin)
339 unpin_pages(&args);
340 /* Succeed */
341 exit(0);
342 }
343 waitpid(child, &result, 0);
344
345 if (with_event) {
346 if (pthread_join(thread, NULL))
347 err("pthread_join()");
348 if (args.child_uffd < 0)
349 err("Didn't receive child uffd");
350 close(args.child_uffd);
351 }
352
353 return result;
354}
355
356static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
357{
358 uint64_t value;
359 int pagemap_fd;
360
361 if (uffd_register(uffd, area_dst, nr_pages * page_size,
362 false, true, false))
363 err("register failed");
364
365 pagemap_fd = pagemap_open();
366
367 /* Test applying pte marker to anon unpopulated */
368 wp_range(uffd, (uint64_t)area_dst, page_size, true);
369 value = pagemap_get_entry(pagemap_fd, area_dst);
370 pagemap_check_wp(value, true);
371
372 /* Test unprotect on anon pte marker */
373 wp_range(uffd, (uint64_t)area_dst, page_size, false);
374 value = pagemap_get_entry(pagemap_fd, area_dst);
375 pagemap_check_wp(value, false);
376
377 /* Test zap on anon marker */
378 wp_range(uffd, (uint64_t)area_dst, page_size, true);
379 if (madvise(area_dst, page_size, MADV_DONTNEED))
380 err("madvise(MADV_DONTNEED) failed");
381 value = pagemap_get_entry(pagemap_fd, area_dst);
382 pagemap_check_wp(value, false);
383
384 /* Test fault in after marker removed */
385 *area_dst = 1;
386 value = pagemap_get_entry(pagemap_fd, area_dst);
387 pagemap_check_wp(value, false);
388 /* Drop it to make pte none again */
389 if (madvise(area_dst, page_size, MADV_DONTNEED))
390 err("madvise(MADV_DONTNEED) failed");
391
392 /* Test read-zero-page upon pte marker */
393 wp_range(uffd, (uint64_t)area_dst, page_size, true);
394 *(volatile char *)area_dst;
395 /* Drop it to make pte none again */
396 if (madvise(area_dst, page_size, MADV_DONTNEED))
397 err("madvise(MADV_DONTNEED) failed");
398
399 uffd_test_pass();
400}
401
402static void uffd_wp_fork_test_common(uffd_test_args_t *args,
403 bool with_event)
404{
405 int pagemap_fd;
406 uint64_t value;
407
408 if (uffd_register(uffd, area_dst, nr_pages * page_size,
409 false, true, false))
410 err("register failed");
411
412 pagemap_fd = pagemap_open();
413
414 /* Touch the page */
415 *area_dst = 1;
416 wp_range(uffd, (uint64_t)area_dst, page_size, true);
417 value = pagemap_get_entry(pagemap_fd, area_dst);
418 pagemap_check_wp(value, true);
419 if (pagemap_test_fork(uffd, with_event, false)) {
420 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
421 with_event ? "missing" : "stall");
422 goto out;
423 }
424
425 /*
426 * This is an attempt for zapping the pgtable so as to test the
427 * markers.
428 *
429 * For private mappings, PAGEOUT will only work on exclusive ptes
430 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
431 *
432 * For shared, PAGEOUT may not work. Use DONTNEED instead which
433 * plays a similar role of zapping (rather than freeing the page)
434 * to expose pte markers.
435 */
436 if (args->mem_type->shared) {
437 if (madvise(area_dst, page_size, MADV_DONTNEED))
438 err("MADV_DONTNEED");
439 } else {
440 /*
441 * NOTE: ignore retval because private-hugetlb doesn't yet
442 * support swapping, so it could fail.
443 */
444 madvise(area_dst, page_size, MADV_PAGEOUT);
445 }
446
447 /* Uffd-wp should persist even swapped out */
448 value = pagemap_get_entry(pagemap_fd, area_dst);
449 pagemap_check_wp(value, true);
450 if (pagemap_test_fork(uffd, with_event, false)) {
451 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
452 with_event ? "missing" : "stall");
453 goto out;
454 }
455
456 /* Unprotect; this tests swap pte modifications */
457 wp_range(uffd, (uint64_t)area_dst, page_size, false);
458 value = pagemap_get_entry(pagemap_fd, area_dst);
459 pagemap_check_wp(value, false);
460
461 /* Fault in the page from disk */
462 *area_dst = 2;
463 value = pagemap_get_entry(pagemap_fd, area_dst);
464 pagemap_check_wp(value, false);
465 uffd_test_pass();
466out:
467 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
468 err("unregister failed");
469 close(pagemap_fd);
470}
471
472static void uffd_wp_fork_test(uffd_test_args_t *args)
473{
474 uffd_wp_fork_test_common(args, false);
475}
476
477static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
478{
479 uffd_wp_fork_test_common(args, true);
480}
481
482static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
483 bool with_event)
484{
485 int pagemap_fd;
486 pin_args pin_args = {};
487
488 if (uffd_register(uffd, area_dst, page_size, false, true, false))
489 err("register failed");
490
491 pagemap_fd = pagemap_open();
492
493 /* Touch the page */
494 *area_dst = 1;
495 wp_range(uffd, (uint64_t)area_dst, page_size, true);
496
497 /*
498 * 1. First pin, then fork(). This tests fork() special path when
499 * doing early CoW if the page is private.
500 */
501 if (pin_pages(&pin_args, area_dst, page_size)) {
502 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
503 "or unprivileged");
504 close(pagemap_fd);
505 uffd_unregister(uffd, area_dst, page_size);
506 return;
507 }
508
509 if (pagemap_test_fork(uffd, with_event, false)) {
510 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
511 with_event ? "missing" : "stall");
512 unpin_pages(&pin_args);
513 goto out;
514 }
515
516 unpin_pages(&pin_args);
517
518 /*
519 * 2. First fork(), then pin (in the child, where test_pin==true).
520 * This tests COR, aka, page unsharing on private memories.
521 */
522 if (pagemap_test_fork(uffd, with_event, true)) {
523 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
524 with_event ? "missing" : "stall");
525 goto out;
526 }
527 uffd_test_pass();
528out:
529 if (uffd_unregister(uffd, area_dst, page_size))
530 err("register failed");
531 close(pagemap_fd);
532}
533
534static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
535{
536 uffd_wp_fork_pin_test_common(args, false);
537}
538
539static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
540{
541 uffd_wp_fork_pin_test_common(args, true);
542}
543
544static void check_memory_contents(char *p)
545{
546 unsigned long i, j;
547 uint8_t expected_byte;
548
549 for (i = 0; i < nr_pages; ++i) {
550 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
551 for (j = 0; j < page_size; j++) {
552 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
553 if (v != expected_byte)
554 err("unexpected page contents");
555 }
556 }
557}
558
559static void uffd_minor_test_common(bool test_collapse, bool test_wp)
560{
561 unsigned long p;
562 pthread_t uffd_mon;
563 char c;
564 struct uffd_args args = { 0 };
565
566 /*
567 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
568 * both do not make much sense.
569 */
570 assert(!(test_collapse && test_wp));
571
572 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
573 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
574 false, test_wp, true))
575 err("register failure");
576
577 /*
578 * After registering with UFFD, populate the non-UFFD-registered side of
579 * the shared mapping. This should *not* trigger any UFFD minor faults.
580 */
581 for (p = 0; p < nr_pages; ++p)
582 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
583 page_size);
584
585 args.apply_wp = test_wp;
586 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
587 err("uffd_poll_thread create");
588
589 /*
590 * Read each of the pages back using the UFFD-registered mapping. We
591 * expect that the first time we touch a page, it will result in a minor
592 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
593 * page's contents, and then issuing a CONTINUE ioctl.
594 */
595 check_memory_contents(area_dst_alias);
596
597 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
598 err("pipe write");
599 if (pthread_join(uffd_mon, NULL))
600 err("join() failed");
601
602 if (test_collapse) {
603 if (madvise(area_dst_alias, nr_pages * page_size,
604 MADV_COLLAPSE)) {
605 /* It's fine to fail for this one... */
606 uffd_test_skip("MADV_COLLAPSE failed");
607 return;
608 }
609
610 uffd_test_ops->check_pmd_mapping(area_dst,
611 nr_pages * page_size /
612 read_pmd_pagesize());
613 /*
614 * This won't cause uffd-fault - it purely just makes sure there
615 * was no corruption.
616 */
617 check_memory_contents(area_dst_alias);
618 }
619
620 if (args.missing_faults != 0 || args.minor_faults != nr_pages)
621 uffd_test_fail("stats check error");
622 else
623 uffd_test_pass();
624}
625
626void uffd_minor_test(uffd_test_args_t *args)
627{
628 uffd_minor_test_common(false, false);
629}
630
631void uffd_minor_wp_test(uffd_test_args_t *args)
632{
633 uffd_minor_test_common(false, true);
634}
635
636void uffd_minor_collapse_test(uffd_test_args_t *args)
637{
638 uffd_minor_test_common(true, false);
639}
640
641static sigjmp_buf jbuf, *sigbuf;
642
643static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
644{
645 if (sig == SIGBUS) {
646 if (sigbuf)
647 siglongjmp(*sigbuf, 1);
648 abort();
649 }
650}
651
652/*
653 * For non-cooperative userfaultfd test we fork() a process that will
654 * generate pagefaults, will mremap the area monitored by the
655 * userfaultfd and at last this process will release the monitored
656 * area.
657 * For the anonymous and shared memory the area is divided into two
658 * parts, the first part is accessed before mremap, and the second
659 * part is accessed after mremap. Since hugetlbfs does not support
660 * mremap, the entire monitored area is accessed in a single pass for
661 * HUGETLB_TEST.
662 * The release of the pages currently generates event for shmem and
663 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
664 * for hugetlb.
665 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
666 * monitored area, generate pagefaults and test that signal is delivered.
667 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
668 * test robustness use case - we release monitored area, fork a process
669 * that will generate pagefaults and verify signal is generated.
670 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
671 * feature. Using monitor thread, verify no userfault events are generated.
672 */
673static int faulting_process(int signal_test, bool wp)
674{
675 unsigned long nr, i;
676 unsigned long long count;
677 unsigned long split_nr_pages;
678 unsigned long lastnr;
679 struct sigaction act;
680 volatile unsigned long signalled = 0;
681
682 split_nr_pages = (nr_pages + 1) / 2;
683
684 if (signal_test) {
685 sigbuf = &jbuf;
686 memset(&act, 0, sizeof(act));
687 act.sa_sigaction = sighndl;
688 act.sa_flags = SA_SIGINFO;
689 if (sigaction(SIGBUS, &act, 0))
690 err("sigaction");
691 lastnr = (unsigned long)-1;
692 }
693
694 for (nr = 0; nr < split_nr_pages; nr++) {
695 volatile int steps = 1;
696 unsigned long offset = nr * page_size;
697
698 if (signal_test) {
699 if (sigsetjmp(*sigbuf, 1) != 0) {
700 if (steps == 1 && nr == lastnr)
701 err("Signal repeated");
702
703 lastnr = nr;
704 if (signal_test == 1) {
705 if (steps == 1) {
706 /* This is a MISSING request */
707 steps++;
708 if (copy_page(uffd, offset, wp))
709 signalled++;
710 } else {
711 /* This is a WP request */
712 assert(steps == 2);
713 wp_range(uffd,
714 (__u64)area_dst +
715 offset,
716 page_size, false);
717 }
718 } else {
719 signalled++;
720 continue;
721 }
722 }
723 }
724
725 count = *area_count(area_dst, nr);
726 if (count != count_verify[nr])
727 err("nr %lu memory corruption %llu %llu\n",
728 nr, count, count_verify[nr]);
729 /*
730 * Trigger write protection if there is by writing
731 * the same value back.
732 */
733 *area_count(area_dst, nr) = count;
734 }
735
736 if (signal_test)
737 return signalled != split_nr_pages;
738
739 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
740 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
741 if (area_dst == MAP_FAILED)
742 err("mremap");
743 /* Reset area_src since we just clobbered it */
744 area_src = NULL;
745
746 for (; nr < nr_pages; nr++) {
747 count = *area_count(area_dst, nr);
748 if (count != count_verify[nr]) {
749 err("nr %lu memory corruption %llu %llu\n",
750 nr, count, count_verify[nr]);
751 }
752 /*
753 * Trigger write protection if there is by writing
754 * the same value back.
755 */
756 *area_count(area_dst, nr) = count;
757 }
758
759 uffd_test_ops->release_pages(area_dst);
760
761 for (nr = 0; nr < nr_pages; nr++)
762 for (i = 0; i < page_size; i++)
763 if (*(area_dst + nr * page_size + i) != 0)
764 err("page %lu offset %lu is not zero", nr, i);
765
766 return 0;
767}
768
769static void uffd_sigbus_test_common(bool wp)
770{
771 unsigned long userfaults;
772 pthread_t uffd_mon;
773 pid_t pid;
774 int err;
775 char c;
776 struct uffd_args args = { 0 };
777
778 ready_for_fork = false;
779
780 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
781
782 if (uffd_register(uffd, area_dst, nr_pages * page_size,
783 true, wp, false))
784 err("register failure");
785
786 if (faulting_process(1, wp))
787 err("faulting process failed");
788
789 uffd_test_ops->release_pages(area_dst);
790
791 args.apply_wp = wp;
792 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
793 err("uffd_poll_thread create");
794
795 while (!ready_for_fork)
796 ; /* Wait for the poll_thread to start executing before forking */
797
798 pid = fork();
799 if (pid < 0)
800 err("fork");
801
802 if (!pid)
803 exit(faulting_process(2, wp));
804
805 waitpid(pid, &err, 0);
806 if (err)
807 err("faulting process failed");
808 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
809 err("pipe write");
810 if (pthread_join(uffd_mon, (void **)&userfaults))
811 err("pthread_join()");
812
813 if (userfaults)
814 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
815 else
816 uffd_test_pass();
817}
818
819static void uffd_sigbus_test(uffd_test_args_t *args)
820{
821 uffd_sigbus_test_common(false);
822}
823
824static void uffd_sigbus_wp_test(uffd_test_args_t *args)
825{
826 uffd_sigbus_test_common(true);
827}
828
829static void uffd_events_test_common(bool wp)
830{
831 pthread_t uffd_mon;
832 pid_t pid;
833 int err;
834 char c;
835 struct uffd_args args = { 0 };
836
837 ready_for_fork = false;
838
839 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
840 if (uffd_register(uffd, area_dst, nr_pages * page_size,
841 true, wp, false))
842 err("register failure");
843
844 args.apply_wp = wp;
845 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
846 err("uffd_poll_thread create");
847
848 while (!ready_for_fork)
849 ; /* Wait for the poll_thread to start executing before forking */
850
851 pid = fork();
852 if (pid < 0)
853 err("fork");
854
855 if (!pid)
856 exit(faulting_process(0, wp));
857
858 waitpid(pid, &err, 0);
859 if (err)
860 err("faulting process failed");
861 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
862 err("pipe write");
863 if (pthread_join(uffd_mon, NULL))
864 err("pthread_join()");
865
866 if (args.missing_faults != nr_pages)
867 uffd_test_fail("Fault counts wrong");
868 else
869 uffd_test_pass();
870}
871
872static void uffd_events_test(uffd_test_args_t *args)
873{
874 uffd_events_test_common(false);
875}
876
877static void uffd_events_wp_test(uffd_test_args_t *args)
878{
879 uffd_events_test_common(true);
880}
881
882static void retry_uffdio_zeropage(int ufd,
883 struct uffdio_zeropage *uffdio_zeropage)
884{
885 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
886 uffdio_zeropage->range.len,
887 0);
888 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
889 if (uffdio_zeropage->zeropage != -EEXIST)
890 err("UFFDIO_ZEROPAGE error: %"PRId64,
891 (int64_t)uffdio_zeropage->zeropage);
892 } else {
893 err("UFFDIO_ZEROPAGE error: %"PRId64,
894 (int64_t)uffdio_zeropage->zeropage);
895 }
896}
897
898static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
899{
900 struct uffdio_zeropage uffdio_zeropage = { 0 };
901 int ret;
902 __s64 res;
903
904 uffdio_zeropage.range.start = (unsigned long) area_dst;
905 uffdio_zeropage.range.len = page_size;
906 uffdio_zeropage.mode = 0;
907 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
908 res = uffdio_zeropage.zeropage;
909 if (ret) {
910 /* real retval in ufdio_zeropage.zeropage */
911 if (has_zeropage)
912 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
913 else if (res != -EINVAL)
914 err("UFFDIO_ZEROPAGE not -EINVAL");
915 } else if (has_zeropage) {
916 if (res != page_size)
917 err("UFFDIO_ZEROPAGE unexpected size");
918 else
919 retry_uffdio_zeropage(ufd, &uffdio_zeropage);
920 return true;
921 } else
922 err("UFFDIO_ZEROPAGE succeeded");
923
924 return false;
925}
926
927/*
928 * Registers a range with MISSING mode only for zeropage test. Return true
929 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
930 * because we want to detect .ioctls along the way.
931 */
932static bool
933uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
934{
935 uint64_t ioctls = 0;
936
937 if (uffd_register_with_ioctls(uffd, addr, len, true,
938 false, false, &ioctls))
939 err("zeropage register fail");
940
941 return ioctls & (1 << _UFFDIO_ZEROPAGE);
942}
943
944/* exercise UFFDIO_ZEROPAGE */
945static void uffd_zeropage_test(uffd_test_args_t *args)
946{
947 bool has_zeropage;
948 int i;
949
950 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
951 if (area_dst_alias)
952 /* Ignore the retval; we already have it */
953 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
954
955 if (do_uffdio_zeropage(uffd, has_zeropage))
956 for (i = 0; i < page_size; i++)
957 if (area_dst[i] != 0)
958 err("data non-zero at offset %d\n", i);
959
960 if (uffd_unregister(uffd, area_dst, page_size))
961 err("unregister");
962
963 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
964 err("unregister");
965
966 uffd_test_pass();
967}
968
969static void uffd_register_poison(int uffd, void *addr, uint64_t len)
970{
971 uint64_t ioctls = 0;
972 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
973
974 if (uffd_register_with_ioctls(uffd, addr, len, true,
975 false, false, &ioctls))
976 err("poison register fail");
977
978 if ((ioctls & expected) != expected)
979 err("registered area doesn't support COPY and POISON ioctls");
980}
981
982static void do_uffdio_poison(int uffd, unsigned long offset)
983{
984 struct uffdio_poison uffdio_poison = { 0 };
985 int ret;
986 __s64 res;
987
988 uffdio_poison.range.start = (unsigned long) area_dst + offset;
989 uffdio_poison.range.len = page_size;
990 uffdio_poison.mode = 0;
991 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
992 res = uffdio_poison.updated;
993
994 if (ret)
995 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
996 else if (res != page_size)
997 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
998}
999
1000static void uffd_poison_handle_fault(
1001 struct uffd_msg *msg, struct uffd_args *args)
1002{
1003 unsigned long offset;
1004
1005 if (msg->event != UFFD_EVENT_PAGEFAULT)
1006 err("unexpected msg event %u", msg->event);
1007
1008 if (msg->arg.pagefault.flags &
1009 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1010 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1011
1012 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1013 offset &= ~(page_size-1);
1014
1015 /* Odd pages -> copy zeroed page; even pages -> poison. */
1016 if (offset & page_size)
1017 copy_page(uffd, offset, false);
1018 else
1019 do_uffdio_poison(uffd, offset);
1020}
1021
1022static void uffd_poison_test(uffd_test_args_t *targs)
1023{
1024 pthread_t uffd_mon;
1025 char c;
1026 struct uffd_args args = { 0 };
1027 struct sigaction act = { 0 };
1028 unsigned long nr_sigbus = 0;
1029 unsigned long nr;
1030
1031 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1032
1033 uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1034 memset(area_src, 0, nr_pages * page_size);
1035
1036 args.handle_fault = uffd_poison_handle_fault;
1037 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1038 err("uffd_poll_thread create");
1039
1040 sigbuf = &jbuf;
1041 act.sa_sigaction = sighndl;
1042 act.sa_flags = SA_SIGINFO;
1043 if (sigaction(SIGBUS, &act, 0))
1044 err("sigaction");
1045
1046 for (nr = 0; nr < nr_pages; ++nr) {
1047 unsigned long offset = nr * page_size;
1048 const char *bytes = (const char *) area_dst + offset;
1049 const char *i;
1050
1051 if (sigsetjmp(*sigbuf, 1)) {
1052 /*
1053 * Access below triggered a SIGBUS, which was caught by
1054 * sighndl, which then jumped here. Count this SIGBUS,
1055 * and move on to next page.
1056 */
1057 ++nr_sigbus;
1058 continue;
1059 }
1060
1061 for (i = bytes; i < bytes + page_size; ++i) {
1062 if (*i)
1063 err("nonzero byte in area_dst (%p) at %p: %u",
1064 area_dst, i, *i);
1065 }
1066 }
1067
1068 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1069 err("pipe write");
1070 if (pthread_join(uffd_mon, NULL))
1071 err("pthread_join()");
1072
1073 if (nr_sigbus != nr_pages / 2)
1074 err("expected to receive %lu SIGBUS, actually received %lu",
1075 nr_pages / 2, nr_sigbus);
1076
1077 uffd_test_pass();
1078}
1079
1080static void
1081uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1082 unsigned long len)
1083{
1084 unsigned long offset;
1085
1086 if (msg->event != UFFD_EVENT_PAGEFAULT)
1087 err("unexpected msg event %u", msg->event);
1088
1089 if (msg->arg.pagefault.flags &
1090 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1091 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1092
1093 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1094 offset &= ~(len-1);
1095
1096 if (move_page(uffd, offset, len))
1097 args->missing_faults++;
1098}
1099
1100static void uffd_move_handle_fault(struct uffd_msg *msg,
1101 struct uffd_args *args)
1102{
1103 uffd_move_handle_fault_common(msg, args, page_size);
1104}
1105
1106static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1107 struct uffd_args *args)
1108{
1109 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1110}
1111
1112static void
1113uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1114 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1115{
1116 unsigned long nr;
1117 pthread_t uffd_mon;
1118 char c;
1119 unsigned long long count;
1120 struct uffd_args args = { 0 };
1121 char *orig_area_src, *orig_area_dst;
1122 unsigned long step_size, step_count;
1123 unsigned long src_offs = 0;
1124 unsigned long dst_offs = 0;
1125
1126 /* Prevent source pages from being mapped more than once */
1127 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1128 err("madvise(MADV_DONTFORK) failure");
1129
1130 if (uffd_register(uffd, area_dst, nr_pages * page_size,
1131 true, false, false))
1132 err("register failure");
1133
1134 args.handle_fault = handle_fault;
1135 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1136 err("uffd_poll_thread create");
1137
1138 step_size = chunk_size / page_size;
1139 step_count = nr_pages / step_size;
1140
1141 if (chunk_size > page_size) {
1142 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1143 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1144
1145 if (aligned_src != area_src || aligned_dst != area_dst) {
1146 src_offs = (aligned_src - area_src) / page_size;
1147 dst_offs = (aligned_dst - area_dst) / page_size;
1148 step_count--;
1149 }
1150 orig_area_src = area_src;
1151 orig_area_dst = area_dst;
1152 area_src = aligned_src;
1153 area_dst = aligned_dst;
1154 }
1155
1156 /*
1157 * Read each of the pages back using the UFFD-registered mapping. We
1158 * expect that the first time we touch a page, it will result in a missing
1159 * fault. uffd_poll_thread will resolve the fault by moving source
1160 * page to destination.
1161 */
1162 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1163 unsigned long i;
1164
1165 /* Check area_src content */
1166 for (i = 0; i < step_size; i++) {
1167 count = *area_count(area_src, nr + i);
1168 if (count != count_verify[src_offs + nr + i])
1169 err("nr %lu source memory invalid %llu %llu\n",
1170 nr + i, count, count_verify[src_offs + nr + i]);
1171 }
1172
1173 /* Faulting into area_dst should move the page or the huge page */
1174 for (i = 0; i < step_size; i++) {
1175 count = *area_count(area_dst, nr + i);
1176 if (count != count_verify[dst_offs + nr + i])
1177 err("nr %lu memory corruption %llu %llu\n",
1178 nr, count, count_verify[dst_offs + nr + i]);
1179 }
1180
1181 /* Re-check area_src content which should be empty */
1182 for (i = 0; i < step_size; i++) {
1183 count = *area_count(area_src, nr + i);
1184 if (count != 0)
1185 err("nr %lu move failed %llu %llu\n",
1186 nr, count, count_verify[src_offs + nr + i]);
1187 }
1188 }
1189 if (step_size > page_size) {
1190 area_src = orig_area_src;
1191 area_dst = orig_area_dst;
1192 }
1193
1194 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1195 err("pipe write");
1196 if (pthread_join(uffd_mon, NULL))
1197 err("join() failed");
1198
1199 if (args.missing_faults != step_count || args.minor_faults != 0)
1200 uffd_test_fail("stats check error");
1201 else
1202 uffd_test_pass();
1203}
1204
1205static void uffd_move_test(uffd_test_args_t *targs)
1206{
1207 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1208}
1209
1210static void uffd_move_pmd_test(uffd_test_args_t *targs)
1211{
1212 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1213 err("madvise(MADV_HUGEPAGE) failure");
1214 uffd_move_test_common(targs, read_pmd_pagesize(),
1215 uffd_move_pmd_handle_fault);
1216}
1217
1218static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1219{
1220 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1221 err("madvise(MADV_NOHUGEPAGE) failure");
1222 uffd_move_test_common(targs, read_pmd_pagesize(),
1223 uffd_move_pmd_handle_fault);
1224}
1225
1226static int prevent_hugepages(const char **errmsg)
1227{
1228 /* This should be done before source area is populated */
1229 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1230 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1231 if (errno != EINVAL) {
1232 if (errmsg)
1233 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1234 return -errno;
1235 }
1236 }
1237 return 0;
1238}
1239
1240static int request_hugepages(const char **errmsg)
1241{
1242 /* This should be done before source area is populated */
1243 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1244 if (errmsg) {
1245 *errmsg = (errno == EINVAL) ?
1246 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1247 "madvise(MADV_HUGEPAGE) failed";
1248 }
1249 return -errno;
1250 }
1251 return 0;
1252}
1253
1254struct uffd_test_case_ops uffd_move_test_case_ops = {
1255 .post_alloc = prevent_hugepages,
1256};
1257
1258struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1259 .post_alloc = request_hugepages,
1260};
1261
1262/*
1263 * Test the returned uffdio_register.ioctls with different register modes.
1264 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1265 */
1266static void
1267do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1268{
1269 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1270 mem_type_t *mem_type = args->mem_type;
1271 int ret;
1272
1273 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1274 miss, wp, minor, &ioctls);
1275
1276 /*
1277 * Handle special cases of UFFDIO_REGISTER here where it should
1278 * just fail with -EINVAL first..
1279 *
1280 * Case 1: register MINOR on anon
1281 * Case 2: register with no mode selected
1282 */
1283 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1284 (!miss && !wp && !minor)) {
1285 if (ret != -EINVAL)
1286 err("register (miss=%d, wp=%d, minor=%d) failed "
1287 "with wrong errno=%d", miss, wp, minor, ret);
1288 return;
1289 }
1290
1291 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1292 if (miss)
1293 expected |= BIT_ULL(_UFFDIO_COPY);
1294 if (wp)
1295 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1296 if (minor)
1297 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1298
1299 if ((ioctls & expected) != expected)
1300 err("unexpected uffdio_register.ioctls "
1301 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1302 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1303
1304 if (uffd_unregister(uffd, area_dst, page_size))
1305 err("unregister");
1306}
1307
1308static void uffd_register_ioctls_test(uffd_test_args_t *args)
1309{
1310 int miss, wp, minor;
1311
1312 for (miss = 0; miss <= 1; miss++)
1313 for (wp = 0; wp <= 1; wp++)
1314 for (minor = 0; minor <= 1; minor++)
1315 do_register_ioctls_test(args, miss, wp, minor);
1316
1317 uffd_test_pass();
1318}
1319
1320uffd_test_case_t uffd_tests[] = {
1321 {
1322 /* Test returned uffdio_register.ioctls. */
1323 .name = "register-ioctls",
1324 .uffd_fn = uffd_register_ioctls_test,
1325 .mem_targets = MEM_ALL,
1326 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1327 UFFD_FEATURE_MISSING_SHMEM |
1328 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1329 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1330 UFFD_FEATURE_MINOR_HUGETLBFS |
1331 UFFD_FEATURE_MINOR_SHMEM,
1332 },
1333 {
1334 .name = "zeropage",
1335 .uffd_fn = uffd_zeropage_test,
1336 .mem_targets = MEM_ALL,
1337 .uffd_feature_required = 0,
1338 },
1339 {
1340 .name = "move",
1341 .uffd_fn = uffd_move_test,
1342 .mem_targets = MEM_ANON,
1343 .uffd_feature_required = UFFD_FEATURE_MOVE,
1344 .test_case_ops = &uffd_move_test_case_ops,
1345 },
1346 {
1347 .name = "move-pmd",
1348 .uffd_fn = uffd_move_pmd_test,
1349 .mem_targets = MEM_ANON,
1350 .uffd_feature_required = UFFD_FEATURE_MOVE,
1351 .test_case_ops = &uffd_move_test_pmd_case_ops,
1352 },
1353 {
1354 .name = "move-pmd-split",
1355 .uffd_fn = uffd_move_pmd_split_test,
1356 .mem_targets = MEM_ANON,
1357 .uffd_feature_required = UFFD_FEATURE_MOVE,
1358 .test_case_ops = &uffd_move_test_pmd_case_ops,
1359 },
1360 {
1361 .name = "wp-fork",
1362 .uffd_fn = uffd_wp_fork_test,
1363 .mem_targets = MEM_ALL,
1364 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1365 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1366 },
1367 {
1368 .name = "wp-fork-with-event",
1369 .uffd_fn = uffd_wp_fork_with_event_test,
1370 .mem_targets = MEM_ALL,
1371 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1372 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1373 /* when set, child process should inherit uffd-wp bits */
1374 UFFD_FEATURE_EVENT_FORK,
1375 },
1376 {
1377 .name = "wp-fork-pin",
1378 .uffd_fn = uffd_wp_fork_pin_test,
1379 .mem_targets = MEM_ALL,
1380 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1381 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1382 },
1383 {
1384 .name = "wp-fork-pin-with-event",
1385 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1386 .mem_targets = MEM_ALL,
1387 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1388 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1389 /* when set, child process should inherit uffd-wp bits */
1390 UFFD_FEATURE_EVENT_FORK,
1391 },
1392 {
1393 .name = "wp-unpopulated",
1394 .uffd_fn = uffd_wp_unpopulated_test,
1395 .mem_targets = MEM_ANON,
1396 .uffd_feature_required =
1397 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1398 },
1399 {
1400 .name = "minor",
1401 .uffd_fn = uffd_minor_test,
1402 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1403 .uffd_feature_required =
1404 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1405 },
1406 {
1407 .name = "minor-wp",
1408 .uffd_fn = uffd_minor_wp_test,
1409 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1410 .uffd_feature_required =
1411 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1412 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1413 /*
1414 * HACK: here we leveraged WP_UNPOPULATED to detect whether
1415 * minor mode supports wr-protect. There's no feature flag
1416 * for it so this is the best we can test against.
1417 */
1418 UFFD_FEATURE_WP_UNPOPULATED,
1419 },
1420 {
1421 .name = "minor-collapse",
1422 .uffd_fn = uffd_minor_collapse_test,
1423 /* MADV_COLLAPSE only works with shmem */
1424 .mem_targets = MEM_SHMEM,
1425 /* We can't test MADV_COLLAPSE, so try our luck */
1426 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1427 },
1428 {
1429 .name = "sigbus",
1430 .uffd_fn = uffd_sigbus_test,
1431 .mem_targets = MEM_ALL,
1432 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1433 UFFD_FEATURE_EVENT_FORK,
1434 },
1435 {
1436 .name = "sigbus-wp",
1437 .uffd_fn = uffd_sigbus_wp_test,
1438 .mem_targets = MEM_ALL,
1439 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1440 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1441 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1442 },
1443 {
1444 .name = "events",
1445 .uffd_fn = uffd_events_test,
1446 .mem_targets = MEM_ALL,
1447 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1448 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1449 },
1450 {
1451 .name = "events-wp",
1452 .uffd_fn = uffd_events_wp_test,
1453 .mem_targets = MEM_ALL,
1454 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1455 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1456 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1457 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1458 },
1459 {
1460 .name = "poison",
1461 .uffd_fn = uffd_poison_test,
1462 .mem_targets = MEM_ALL,
1463 .uffd_feature_required = UFFD_FEATURE_POISON,
1464 },
1465};
1466
1467static void usage(const char *prog)
1468{
1469 printf("usage: %s [-f TESTNAME]\n", prog);
1470 puts("");
1471 puts(" -f: test name to filter (e.g., event)");
1472 puts(" -h: show the help msg");
1473 puts(" -l: list tests only");
1474 puts("");
1475 exit(KSFT_FAIL);
1476}
1477
1478int main(int argc, char *argv[])
1479{
1480 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1481 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1482 const char *test_filter = NULL;
1483 bool list_only = false;
1484 uffd_test_case_t *test;
1485 mem_type_t *mem_type;
1486 uffd_test_args_t args;
1487 const char *errmsg;
1488 int has_uffd, opt;
1489 int i, j;
1490
1491 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1492 switch (opt) {
1493 case 'f':
1494 test_filter = optarg;
1495 break;
1496 case 'l':
1497 list_only = true;
1498 break;
1499 case 'h':
1500 default:
1501 /* Unknown */
1502 usage(argv[0]);
1503 break;
1504 }
1505 }
1506
1507 if (!test_filter && !list_only) {
1508 has_uffd = test_uffd_api(false);
1509 has_uffd |= test_uffd_api(true);
1510
1511 if (!has_uffd) {
1512 printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1513 exit(KSFT_SKIP);
1514 }
1515 }
1516
1517 for (i = 0; i < n_tests; i++) {
1518 test = &uffd_tests[i];
1519 if (test_filter && !strstr(test->name, test_filter))
1520 continue;
1521 if (list_only) {
1522 printf("%s\n", test->name);
1523 continue;
1524 }
1525 for (j = 0; j < n_mems; j++) {
1526 mem_type = &mem_types[j];
1527 if (!(test->mem_targets & mem_type->mem_flag))
1528 continue;
1529
1530 uffd_test_start("%s on %s", test->name, mem_type->name);
1531 if ((mem_type->mem_flag == MEM_HUGETLB ||
1532 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1533 (default_huge_page_size() == 0)) {
1534 uffd_test_skip("huge page size is 0, feature missing?");
1535 continue;
1536 }
1537 if (!uffd_feature_supported(test)) {
1538 uffd_test_skip("feature missing");
1539 continue;
1540 }
1541 if (uffd_setup_environment(&args, test, mem_type,
1542 &errmsg)) {
1543 uffd_test_skip(errmsg);
1544 continue;
1545 }
1546 test->uffd_fn(&args);
1547 uffd_test_ctx_clear();
1548 }
1549 }
1550
1551 if (!list_only)
1552 uffd_test_report();
1553
1554 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1555}
1556
1557#else /* __NR_userfaultfd */
1558
1559#warning "missing __NR_userfaultfd definition"
1560
1561int main(void)
1562{
1563 printf("Skipping %s (missing __NR_userfaultfd)\n", __file__);
1564 return KSFT_SKIP;
1565}
1566
1567#endif /* __NR_userfaultfd */