Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2020 Google LLC
4 */
5#define _GNU_SOURCE
6
7#include <errno.h>
8#include <stdlib.h>
9#include <stdio.h>
10#include <string.h>
11#include <sys/mman.h>
12#include <time.h>
13#include <stdbool.h>
14
15#include "../kselftest.h"
16
17#define EXPECT_SUCCESS 0
18#define EXPECT_FAILURE 1
19#define NON_OVERLAPPING 0
20#define OVERLAPPING 1
21#define NS_PER_SEC 1000000000ULL
22#define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */
23#define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */
24
25#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
26
27struct config {
28 unsigned long long src_alignment;
29 unsigned long long dest_alignment;
30 unsigned long long region_size;
31 int overlapping;
32};
33
34struct test {
35 const char *name;
36 struct config config;
37 int expect_failure;
38};
39
40enum {
41 _1KB = 1ULL << 10, /* 1KB -> not page aligned */
42 _4KB = 4ULL << 10,
43 _8KB = 8ULL << 10,
44 _1MB = 1ULL << 20,
45 _2MB = 2ULL << 20,
46 _4MB = 4ULL << 20,
47 _1GB = 1ULL << 30,
48 _2GB = 2ULL << 30,
49 PMD = _2MB,
50 PUD = _1GB,
51};
52
53#define PTE page_size
54
55#define MAKE_TEST(source_align, destination_align, size, \
56 overlaps, should_fail, test_name) \
57(struct test){ \
58 .name = test_name, \
59 .config = { \
60 .src_alignment = source_align, \
61 .dest_alignment = destination_align, \
62 .region_size = size, \
63 .overlapping = overlaps, \
64 }, \
65 .expect_failure = should_fail \
66}
67
68/*
69 * Returns false if the requested remap region overlaps with an
70 * existing mapping (e.g text, stack) else returns true.
71 */
72static bool is_remap_region_valid(void *addr, unsigned long long size)
73{
74 void *remap_addr = NULL;
75 bool ret = true;
76
77 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
78 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
79 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
80 -1, 0);
81
82 if (remap_addr == MAP_FAILED) {
83 if (errno == EEXIST)
84 ret = false;
85 } else {
86 munmap(remap_addr, size);
87 }
88
89 return ret;
90}
91
92/* Returns mmap_min_addr sysctl tunable from procfs */
93static unsigned long long get_mmap_min_addr(void)
94{
95 FILE *fp;
96 int n_matched;
97 static unsigned long long addr;
98
99 if (addr)
100 return addr;
101
102 fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
103 if (fp == NULL) {
104 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
105 strerror(errno));
106 exit(KSFT_SKIP);
107 }
108
109 n_matched = fscanf(fp, "%llu", &addr);
110 if (n_matched != 1) {
111 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
112 strerror(errno));
113 fclose(fp);
114 exit(KSFT_SKIP);
115 }
116
117 fclose(fp);
118 return addr;
119}
120
121/*
122 * This test validates that merge is called when expanding a mapping.
123 * Mapping containing three pages is created, middle page is unmapped
124 * and then the mapping containing the first page is expanded so that
125 * it fills the created hole. The two parts should merge creating
126 * single mapping with three pages.
127 */
128static void mremap_expand_merge(unsigned long page_size)
129{
130 char *test_name = "mremap expand merge";
131 FILE *fp;
132 char *line = NULL;
133 size_t len = 0;
134 bool success = false;
135 char *start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
136 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
137
138 munmap(start + page_size, page_size);
139 mremap(start, page_size, 2 * page_size, 0);
140
141 fp = fopen("/proc/self/maps", "r");
142 if (fp == NULL) {
143 ksft_test_result_fail("%s\n", test_name);
144 return;
145 }
146
147 while (getline(&line, &len, fp) != -1) {
148 char *first = strtok(line, "- ");
149 void *first_val = (void *)strtol(first, NULL, 16);
150 char *second = strtok(NULL, "- ");
151 void *second_val = (void *) strtol(second, NULL, 16);
152
153 if (first_val == start && second_val == start + 3 * page_size) {
154 success = true;
155 break;
156 }
157 }
158 if (success)
159 ksft_test_result_pass("%s\n", test_name);
160 else
161 ksft_test_result_fail("%s\n", test_name);
162 fclose(fp);
163}
164
165/*
166 * Returns the start address of the mapping on success, else returns
167 * NULL on failure.
168 */
169static void *get_source_mapping(struct config c)
170{
171 unsigned long long addr = 0ULL;
172 void *src_addr = NULL;
173 unsigned long long mmap_min_addr;
174
175 mmap_min_addr = get_mmap_min_addr();
176
177retry:
178 addr += c.src_alignment;
179 if (addr < mmap_min_addr)
180 goto retry;
181
182 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
183 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
184 -1, 0);
185 if (src_addr == MAP_FAILED) {
186 if (errno == EPERM || errno == EEXIST)
187 goto retry;
188 goto error;
189 }
190 /*
191 * Check that the address is aligned to the specified alignment.
192 * Addresses which have alignments that are multiples of that
193 * specified are not considered valid. For instance, 1GB address is
194 * 2MB-aligned, however it will not be considered valid for a
195 * requested alignment of 2MB. This is done to reduce coincidental
196 * alignment in the tests.
197 */
198 if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
199 !((unsigned long long) src_addr & c.src_alignment)) {
200 munmap(src_addr, c.region_size);
201 goto retry;
202 }
203
204 if (!src_addr)
205 goto error;
206
207 return src_addr;
208error:
209 ksft_print_msg("Failed to map source region: %s\n",
210 strerror(errno));
211 return NULL;
212}
213
214/* Returns the time taken for the remap on success else returns -1. */
215static long long remap_region(struct config c, unsigned int threshold_mb,
216 char pattern_seed)
217{
218 void *addr, *src_addr, *dest_addr;
219 unsigned long long i;
220 struct timespec t_start = {0, 0}, t_end = {0, 0};
221 long long start_ns, end_ns, align_mask, ret, offset;
222 unsigned long long threshold;
223
224 if (threshold_mb == VALIDATION_NO_THRESHOLD)
225 threshold = c.region_size;
226 else
227 threshold = MIN(threshold_mb * _1MB, c.region_size);
228
229 src_addr = get_source_mapping(c);
230 if (!src_addr) {
231 ret = -1;
232 goto out;
233 }
234
235 /* Set byte pattern */
236 srand(pattern_seed);
237 for (i = 0; i < threshold; i++)
238 memset((char *) src_addr + i, (char) rand(), 1);
239
240 /* Mask to zero out lower bits of address for alignment */
241 align_mask = ~(c.dest_alignment - 1);
242 /* Offset of destination address from the end of the source region */
243 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
244 addr = (void *) (((unsigned long long) src_addr + c.region_size
245 + offset) & align_mask);
246
247 /* See comment in get_source_mapping() */
248 if (!((unsigned long long) addr & c.dest_alignment))
249 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
250
251 /* Don't destroy existing mappings unless expected to overlap */
252 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
253 /* Check for unsigned overflow */
254 if (addr + c.dest_alignment < addr) {
255 ksft_print_msg("Couldn't find a valid region to remap to\n");
256 ret = -1;
257 goto out;
258 }
259 addr += c.dest_alignment;
260 }
261
262 clock_gettime(CLOCK_MONOTONIC, &t_start);
263 dest_addr = mremap(src_addr, c.region_size, c.region_size,
264 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
265 clock_gettime(CLOCK_MONOTONIC, &t_end);
266
267 if (dest_addr == MAP_FAILED) {
268 ksft_print_msg("mremap failed: %s\n", strerror(errno));
269 ret = -1;
270 goto clean_up_src;
271 }
272
273 /* Verify byte pattern after remapping */
274 srand(pattern_seed);
275 for (i = 0; i < threshold; i++) {
276 char c = (char) rand();
277
278 if (((char *) dest_addr)[i] != c) {
279 ksft_print_msg("Data after remap doesn't match at offset %d\n",
280 i);
281 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
282 ((char *) dest_addr)[i] & 0xff);
283 ret = -1;
284 goto clean_up_dest;
285 }
286 }
287
288 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
289 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
290 ret = end_ns - start_ns;
291
292/*
293 * Since the destination address is specified using MREMAP_FIXED, subsequent
294 * mremap will unmap any previous mapping at the address range specified by
295 * dest_addr and region_size. This significantly affects the remap time of
296 * subsequent tests. So we clean up mappings after each test.
297 */
298clean_up_dest:
299 munmap(dest_addr, c.region_size);
300clean_up_src:
301 munmap(src_addr, c.region_size);
302out:
303 return ret;
304}
305
306static void run_mremap_test_case(struct test test_case, int *failures,
307 unsigned int threshold_mb,
308 unsigned int pattern_seed)
309{
310 long long remap_time = remap_region(test_case.config, threshold_mb,
311 pattern_seed);
312
313 if (remap_time < 0) {
314 if (test_case.expect_failure)
315 ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
316 test_case.name);
317 else {
318 ksft_test_result_fail("%s\n", test_case.name);
319 *failures += 1;
320 }
321 } else {
322 /*
323 * Comparing mremap time is only applicable if entire region
324 * was faulted in.
325 */
326 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
327 test_case.config.region_size <= threshold_mb * _1MB)
328 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
329 test_case.name, remap_time);
330 else
331 ksft_test_result_pass("%s\n", test_case.name);
332 }
333}
334
335static void usage(const char *cmd)
336{
337 fprintf(stderr,
338 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
339 "-t\t only validate threshold_mb of the remapped region\n"
340 " \t if 0 is supplied no threshold is used; all tests\n"
341 " \t are run and remapped regions validated fully.\n"
342 " \t The default threshold used is 4MB.\n"
343 "-p\t provide a seed to generate the random pattern for\n"
344 " \t validating the remapped region.\n", cmd);
345}
346
347static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
348 unsigned int *pattern_seed)
349{
350 const char *optstr = "t:p:";
351 int opt;
352
353 while ((opt = getopt(argc, argv, optstr)) != -1) {
354 switch (opt) {
355 case 't':
356 *threshold_mb = atoi(optarg);
357 break;
358 case 'p':
359 *pattern_seed = atoi(optarg);
360 break;
361 default:
362 usage(argv[0]);
363 return -1;
364 }
365 }
366
367 if (optind < argc) {
368 usage(argv[0]);
369 return -1;
370 }
371
372 return 0;
373}
374
375#define MAX_TEST 13
376#define MAX_PERF_TEST 3
377int main(int argc, char **argv)
378{
379 int failures = 0;
380 int i, run_perf_tests;
381 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
382 unsigned int pattern_seed;
383 int num_expand_tests = 1;
384 struct test test_cases[MAX_TEST];
385 struct test perf_test_cases[MAX_PERF_TEST];
386 int page_size;
387 time_t t;
388
389 pattern_seed = (unsigned int) time(&t);
390
391 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
392 exit(EXIT_FAILURE);
393
394 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
395 threshold_mb, pattern_seed);
396
397 page_size = sysconf(_SC_PAGESIZE);
398
399 /* Expected mremap failures */
400 test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
401 OVERLAPPING, EXPECT_FAILURE,
402 "mremap - Source and Destination Regions Overlapping");
403
404 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
405 NON_OVERLAPPING, EXPECT_FAILURE,
406 "mremap - Destination Address Misaligned (1KB-aligned)");
407 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
408 NON_OVERLAPPING, EXPECT_FAILURE,
409 "mremap - Source Address Misaligned (1KB-aligned)");
410
411 /* Src addr PTE aligned */
412 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
413 NON_OVERLAPPING, EXPECT_SUCCESS,
414 "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
415
416 /* Src addr 1MB aligned */
417 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
418 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
419 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
420 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
421
422 /* Src addr PMD aligned */
423 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
424 "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
425 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
426 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
427 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
428 "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
429
430 /* Src addr PUD aligned */
431 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
432 "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
433 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
434 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
435 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
436 "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
437 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
438 "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
439
440 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
441 "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
442 /*
443 * mremap 1GB region - Page table level aligned time
444 * comparison.
445 */
446 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
447 "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
448 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
449 "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
450
451 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) ||
452 (threshold_mb * _1MB >= _1GB);
453
454 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
455 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests);
456
457 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
458 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
459 pattern_seed);
460
461 mremap_expand_merge(page_size);
462
463 if (run_perf_tests) {
464 ksft_print_msg("\n%s\n",
465 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
466 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
467 run_mremap_test_case(perf_test_cases[i], &failures,
468 threshold_mb, pattern_seed);
469 }
470
471 if (failures > 0)
472 ksft_exit_fail();
473 else
474 ksft_exit_pass();
475}