1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Userfaultfd tests common header
  4 *
  5 * Copyright (C) 2015-2023  Red Hat, Inc.
  6 */
  7#ifndef __UFFD_COMMON_H__
  8#define __UFFD_COMMON_H__
  9
 10#define _GNU_SOURCE
 11#include <stdio.h>
 12#include <errno.h>
 13#include <unistd.h>
 14#include <stdlib.h>
 15#include <sys/types.h>
 16#include <sys/stat.h>
 17#include <fcntl.h>
 18#include <time.h>
 19#include <signal.h>
 20#include <poll.h>
 21#include <string.h>
 22#include <linux/mman.h>
 23#include <sys/mman.h>
 24#include <sys/syscall.h>
 25#include <sys/ioctl.h>
 26#include <sys/wait.h>
 27#include <pthread.h>
 28#include <linux/userfaultfd.h>
 29#include <setjmp.h>
 30#include <stdbool.h>
 31#include <assert.h>
 32#include <inttypes.h>
 33#include <stdint.h>
 34#include <sys/random.h>
 35#include <stdatomic.h>
 36
 37#include "../kselftest.h"
 38#include "vm_util.h"
 39
 40#define UFFD_FLAGS	(O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY)
 41
 42#define _err(fmt, ...)						\
 43	do {							\
 44		int ret = errno;				\
 45		fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__);	\
 46		fprintf(stderr, " (errno=%d, @%s:%d)\n",	\
 47			ret, __FILE__, __LINE__);		\
 48	} while (0)
 49
 50#define errexit(exitcode, fmt, ...)		\
 51	do {					\
 52		_err(fmt, ##__VA_ARGS__);	\
 53		exit(exitcode);			\
 54	} while (0)
 55
 56#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__)
 57
 58/* pthread_mutex_t starts at page offset 0 */
 59#define area_mutex(___area, ___nr)					\
 60	((pthread_mutex_t *) ((___area) + (___nr)*page_size))
 61/*
 62 * count is placed in the page after pthread_mutex_t naturally aligned
 63 * to avoid non alignment faults on non-x86 archs.
 64 */
 65#define area_count(___area, ___nr)					\
 66	((volatile unsigned long long *) ((unsigned long)		\
 67				 ((___area) + (___nr)*page_size +	\
 68				  sizeof(pthread_mutex_t) +		\
 69				  sizeof(unsigned long long) - 1) &	\
 70				 ~(unsigned long)(sizeof(unsigned long long) \
 71						  -  1)))
 72
 73/* Userfaultfd test statistics */
 74struct uffd_args {
 75	int cpu;
 76	/* Whether apply wr-protects when installing pages */
 77	bool apply_wp;
 78	unsigned long missing_faults;
 79	unsigned long wp_faults;
 80	unsigned long minor_faults;
 81
 82	/* A custom fault handler; defaults to uffd_handle_page_fault. */
 83	void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args);
 84};
 85
 86struct uffd_test_ops {
 87	int (*allocate_area)(void **alloc_area, bool is_src);
 88	void (*release_pages)(char *rel_area);
 89	void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
 90	void (*check_pmd_mapping)(void *p, int expect_nr_hpages);
 91};
 92typedef struct uffd_test_ops uffd_test_ops_t;
 93
 94struct uffd_test_case_ops {
 95	int (*pre_alloc)(const char **errmsg);
 96	int (*post_alloc)(const char **errmsg);
 97};
 98typedef struct uffd_test_case_ops uffd_test_case_ops_t;
 99
100extern unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
101extern char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
102extern int uffd, uffd_flags, finished, *pipefd, test_type;
103extern bool map_shared;
104extern bool test_uffdio_wp;
105extern unsigned long long *count_verify;
106extern volatile bool test_uffdio_copy_eexist;
107extern atomic_bool ready_for_fork;
108
109extern uffd_test_ops_t anon_uffd_test_ops;
110extern uffd_test_ops_t shmem_uffd_test_ops;
111extern uffd_test_ops_t hugetlb_uffd_test_ops;
112extern uffd_test_ops_t *uffd_test_ops;
113extern uffd_test_case_ops_t *uffd_test_case_ops;
114
115void uffd_stats_report(struct uffd_args *args, int n_cpus);
116int uffd_test_ctx_init(uint64_t features, const char **errmsg);
117void uffd_test_ctx_clear(void);
118int userfaultfd_open(uint64_t *features);
119int uffd_read_msg(int ufd, struct uffd_msg *msg);
120void wp_range(int ufd, __u64 start, __u64 len, bool wp);
121void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args);
122int __copy_page(int ufd, unsigned long offset, bool retry, bool wp);
123int copy_page(int ufd, unsigned long offset, bool wp);
124int move_page(int ufd, unsigned long offset, unsigned long len);
125void *uffd_poll_thread(void *arg);
126
127int uffd_open_dev(unsigned int flags);
128int uffd_open_sys(unsigned int flags);
129int uffd_open(unsigned int flags);
130int uffd_get_features(uint64_t *features);
131
132#define TEST_ANON	1
133#define TEST_HUGETLB	2
134#define TEST_SHMEM	3
135
136#endif