1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright 2020 Google LLC
  4 */
  5#define _GNU_SOURCE
  6
  7#include <errno.h>
  8#include <stdlib.h>
  9#include <string.h>
 10#include <sys/mman.h>
 11#include <time.h>
 12
 13#include "../kselftest.h"
 14
 15#define EXPECT_SUCCESS 0
 16#define EXPECT_FAILURE 1
 17#define NON_OVERLAPPING 0
 18#define OVERLAPPING 1
 19#define NS_PER_SEC 1000000000ULL
 20#define VALIDATION_DEFAULT_THRESHOLD 4	/* 4MB */
 21#define VALIDATION_NO_THRESHOLD 0	/* Verify the entire region */
 22
 23#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 24#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
 25
 26struct config {
 27	unsigned long long src_alignment;
 28	unsigned long long dest_alignment;
 29	unsigned long long region_size;
 30	int overlapping;
 31};
 32
 33struct test {
 34	const char *name;
 35	struct config config;
 36	int expect_failure;
 37};
 38
 39enum {
 40	_1KB = 1ULL << 10,	/* 1KB -> not page aligned */
 41	_4KB = 4ULL << 10,
 42	_8KB = 8ULL << 10,
 43	_1MB = 1ULL << 20,
 44	_2MB = 2ULL << 20,
 45	_4MB = 4ULL << 20,
 46	_1GB = 1ULL << 30,
 47	_2GB = 2ULL << 30,
 48	PMD = _2MB,
 49	PUD = _1GB,
 50};
 51
 52#define PTE page_size
 53
 54#define MAKE_TEST(source_align, destination_align, size,	\
 55		  overlaps, should_fail, test_name)		\
 56(struct test){							\
 57	.name = test_name,					\
 58	.config = {						\
 59		.src_alignment = source_align,			\
 60		.dest_alignment = destination_align,		\
 61		.region_size = size,				\
 62		.overlapping = overlaps,			\
 63	},							\
 64	.expect_failure = should_fail				\
 65}
 66
 67/*
 68 * Returns the start address of the mapping on success, else returns
 69 * NULL on failure.
 70 */
 71static void *get_source_mapping(struct config c)
 72{
 73	unsigned long long addr = 0ULL;
 74	void *src_addr = NULL;
 75retry:
 76	addr += c.src_alignment;
 77	src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
 78			MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 79			-1, 0);
 80	if (src_addr == MAP_FAILED) {
 81		if (errno == EPERM || errno == EEXIST)
 82			goto retry;
 83		goto error;
 84	}
 85	/*
 86	 * Check that the address is aligned to the specified alignment.
 87	 * Addresses which have alignments that are multiples of that
 88	 * specified are not considered valid. For instance, 1GB address is
 89	 * 2MB-aligned, however it will not be considered valid for a
 90	 * requested alignment of 2MB. This is done to reduce coincidental
 91	 * alignment in the tests.
 92	 */
 93	if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
 94			!((unsigned long long) src_addr & c.src_alignment))
 95		goto retry;
 96
 97	if (!src_addr)
 98		goto error;
 99
100	return src_addr;
101error:
102	ksft_print_msg("Failed to map source region: %s\n",
103			strerror(errno));
104	return NULL;
105}
106
107/* Returns the time taken for the remap on success else returns -1. */
108static long long remap_region(struct config c, unsigned int threshold_mb,
109			      char pattern_seed)
110{
111	void *addr, *src_addr, *dest_addr;
112	unsigned long long i;
113	struct timespec t_start = {0, 0}, t_end = {0, 0};
114	long long  start_ns, end_ns, align_mask, ret, offset;
115	unsigned long long threshold;
116
117	if (threshold_mb == VALIDATION_NO_THRESHOLD)
118		threshold = c.region_size;
119	else
120		threshold = MIN(threshold_mb * _1MB, c.region_size);
121
122	src_addr = get_source_mapping(c);
123	if (!src_addr) {
124		ret = -1;
125		goto out;
126	}
127
128	/* Set byte pattern */
129	srand(pattern_seed);
130	for (i = 0; i < threshold; i++)
131		memset((char *) src_addr + i, (char) rand(), 1);
132
133	/* Mask to zero out lower bits of address for alignment */
134	align_mask = ~(c.dest_alignment - 1);
135	/* Offset of destination address from the end of the source region */
136	offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
137	addr = (void *) (((unsigned long long) src_addr + c.region_size
138			  + offset) & align_mask);
139
140	/* See comment in get_source_mapping() */
141	if (!((unsigned long long) addr & c.dest_alignment))
142		addr = (void *) ((unsigned long long) addr | c.dest_alignment);
143
144	clock_gettime(CLOCK_MONOTONIC, &t_start);
145	dest_addr = mremap(src_addr, c.region_size, c.region_size,
146			MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
147	clock_gettime(CLOCK_MONOTONIC, &t_end);
148
149	if (dest_addr == MAP_FAILED) {
150		ksft_print_msg("mremap failed: %s\n", strerror(errno));
151		ret = -1;
152		goto clean_up_src;
153	}
154
155	/* Verify byte pattern after remapping */
156	srand(pattern_seed);
157	for (i = 0; i < threshold; i++) {
158		char c = (char) rand();
159
160		if (((char *) dest_addr)[i] != c) {
161			ksft_print_msg("Data after remap doesn't match at offset %d\n",
162				       i);
163			ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
164					((char *) dest_addr)[i] & 0xff);
165			ret = -1;
166			goto clean_up_dest;
167		}
168	}
169
170	start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
171	end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
172	ret = end_ns - start_ns;
173
174/*
175 * Since the destination address is specified using MREMAP_FIXED, subsequent
176 * mremap will unmap any previous mapping at the address range specified by
177 * dest_addr and region_size. This significantly affects the remap time of
178 * subsequent tests. So we clean up mappings after each test.
179 */
180clean_up_dest:
181	munmap(dest_addr, c.region_size);
182clean_up_src:
183	munmap(src_addr, c.region_size);
184out:
185	return ret;
186}
187
188static void run_mremap_test_case(struct test test_case, int *failures,
189				 unsigned int threshold_mb,
190				 unsigned int pattern_seed)
191{
192	long long remap_time = remap_region(test_case.config, threshold_mb,
193					    pattern_seed);
194
195	if (remap_time < 0) {
196		if (test_case.expect_failure)
197			ksft_test_result_pass("%s\n\tExpected mremap failure\n",
198					      test_case.name);
199		else {
200			ksft_test_result_fail("%s\n", test_case.name);
201			*failures += 1;
202		}
203	} else {
204		/*
205		 * Comparing mremap time is only applicable if entire region
206		 * was faulted in.
207		 */
208		if (threshold_mb == VALIDATION_NO_THRESHOLD ||
209		    test_case.config.region_size <= threshold_mb * _1MB)
210			ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
211					      test_case.name, remap_time);
212		else
213			ksft_test_result_pass("%s\n", test_case.name);
214	}
215}
216
217static void usage(const char *cmd)
218{
219	fprintf(stderr,
220		"Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
221		"-t\t only validate threshold_mb of the remapped region\n"
222		"  \t if 0 is supplied no threshold is used; all tests\n"
223		"  \t are run and remapped regions validated fully.\n"
224		"  \t The default threshold used is 4MB.\n"
225		"-p\t provide a seed to generate the random pattern for\n"
226		"  \t validating the remapped region.\n", cmd);
227}
228
229static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
230		      unsigned int *pattern_seed)
231{
232	const char *optstr = "t:p:";
233	int opt;
234
235	while ((opt = getopt(argc, argv, optstr)) != -1) {
236		switch (opt) {
237		case 't':
238			*threshold_mb = atoi(optarg);
239			break;
240		case 'p':
241			*pattern_seed = atoi(optarg);
242			break;
243		default:
244			usage(argv[0]);
245			return -1;
246		}
247	}
248
249	if (optind < argc) {
250		usage(argv[0]);
251		return -1;
252	}
253
254	return 0;
255}
256
257#define MAX_TEST 13
258#define MAX_PERF_TEST 3
259int main(int argc, char **argv)
260{
261	int failures = 0;
262	int i, run_perf_tests;
263	unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
264	unsigned int pattern_seed;
265	struct test test_cases[MAX_TEST];
266	struct test perf_test_cases[MAX_PERF_TEST];
267	int page_size;
268	time_t t;
269
270	pattern_seed = (unsigned int) time(&t);
271
272	if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
273		exit(EXIT_FAILURE);
274
275	ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
276		       threshold_mb, pattern_seed);
277
278	page_size = sysconf(_SC_PAGESIZE);
279
280	/* Expected mremap failures */
281	test_cases[0] =	MAKE_TEST(page_size, page_size, page_size,
282				  OVERLAPPING, EXPECT_FAILURE,
283				  "mremap - Source and Destination Regions Overlapping");
284
285	test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
286				  NON_OVERLAPPING, EXPECT_FAILURE,
287				  "mremap - Destination Address Misaligned (1KB-aligned)");
288	test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
289				  NON_OVERLAPPING, EXPECT_FAILURE,
290				  "mremap - Source Address Misaligned (1KB-aligned)");
291
292	/* Src addr PTE aligned */
293	test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
294				  NON_OVERLAPPING, EXPECT_SUCCESS,
295				  "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
296
297	/* Src addr 1MB aligned */
298	test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
299				  "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
300	test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
301				  "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
302
303	/* Src addr PMD aligned */
304	test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
305				  "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
306	test_cases[7] =	MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
307				  "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
308	test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
309				  "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
310
311	/* Src addr PUD aligned */
312	test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
313				  "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
314	test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
315				   "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
316	test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
317				   "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
318	test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
319				   "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
320
321	perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
322					"1GB mremap - Source PTE-aligned, Destination PTE-aligned");
323	/*
324	 * mremap 1GB region - Page table level aligned time
325	 * comparison.
326	 */
327	perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
328				       "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
329	perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
330				       "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
331
332	run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
333				(threshold_mb * _1MB >= _1GB);
334
335	ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
336		      ARRAY_SIZE(perf_test_cases) : 0));
337
338	for (i = 0; i < ARRAY_SIZE(test_cases); i++)
339		run_mremap_test_case(test_cases[i], &failures, threshold_mb,
340				     pattern_seed);
341
342	if (run_perf_tests) {
343		ksft_print_msg("\n%s\n",
344		 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
345		for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
346			run_mremap_test_case(perf_test_cases[i], &failures,
347					     threshold_mb, pattern_seed);
348	}
349
350	if (failures > 0)
351		ksft_exit_fail();
352	else
353		ksft_exit_pass();
354}