1#include <fcntl.h>
  2#include <errno.h>
  3#include <stdio.h>
  4#include <stdlib.h>
  5#include <unistd.h>
  6#include <dirent.h>
  7#include <sys/ioctl.h>
  8#include <sys/mman.h>
  9#include <sys/stat.h>
 10#include <sys/types.h>
 11#include <pthread.h>
 12#include <assert.h>
 13#include <mm/gup_test.h>
 14#include "../kselftest.h"
 15
 16#include "util.h"
 17
 18#define MB (1UL << 20)
 19
 20/* Just the flags we need, copied from mm.h: */
 21#define FOLL_WRITE	0x01	/* check pte is writable */
 22#define FOLL_TOUCH	0x02	/* mark page accessed */
 23
 24#define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
 25
 26static unsigned long cmd = GUP_FAST_BENCHMARK;
 27static int gup_fd, repeats = 1;
 28static unsigned long size = 128 * MB;
 29/* Serialize prints */
 30static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
 31
 32static char *cmd_to_str(unsigned long cmd)
 33{
 34	switch (cmd) {
 35	case GUP_FAST_BENCHMARK:
 36		return "GUP_FAST_BENCHMARK";
 37	case PIN_FAST_BENCHMARK:
 38		return "PIN_FAST_BENCHMARK";
 39	case PIN_LONGTERM_BENCHMARK:
 40		return "PIN_LONGTERM_BENCHMARK";
 41	case GUP_BASIC_TEST:
 42		return "GUP_BASIC_TEST";
 43	case PIN_BASIC_TEST:
 44		return "PIN_BASIC_TEST";
 45	case DUMP_USER_PAGES_TEST:
 46		return "DUMP_USER_PAGES_TEST";
 47	}
 48	return "Unknown command";
 49}
 50
 51void *gup_thread(void *data)
 52{
 53	struct gup_test gup = *(struct gup_test *)data;
 54	int i;
 55
 56	/* Only report timing information on the *_BENCHMARK commands: */
 57	if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
 58	     (cmd == PIN_LONGTERM_BENCHMARK)) {
 59		for (i = 0; i < repeats; i++) {
 60			gup.size = size;
 61			if (ioctl(gup_fd, cmd, &gup))
 62				perror("ioctl"), exit(1);
 63
 64			pthread_mutex_lock(&print_mutex);
 65			printf("%s: Time: get:%lld put:%lld us",
 66			       cmd_to_str(cmd), gup.get_delta_usec,
 67			       gup.put_delta_usec);
 68			if (gup.size != size)
 69				printf(", truncated (size: %lld)", gup.size);
 70			printf("\n");
 71			pthread_mutex_unlock(&print_mutex);
 72		}
 73	} else {
 74		gup.size = size;
 75		if (ioctl(gup_fd, cmd, &gup)) {
 76			perror("ioctl");
 77			exit(1);
 78		}
 79
 80		pthread_mutex_lock(&print_mutex);
 81		printf("%s: done\n", cmd_to_str(cmd));
 82		if (gup.size != size)
 83			printf("Truncated (size: %lld)\n", gup.size);
 84		pthread_mutex_unlock(&print_mutex);
 85	}
 86
 87	return NULL;
 88}
 89
 90int main(int argc, char **argv)
 91{
 92	struct gup_test gup = { 0 };
 93	int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
 94	int flags = MAP_PRIVATE, touch = 0;
 95	char *file = "/dev/zero";
 96	pthread_t *tid;
 97	char *p;
 98
 99	while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
100		switch (opt) {
101		case 'a':
102			cmd = PIN_FAST_BENCHMARK;
103			break;
104		case 'b':
105			cmd = PIN_BASIC_TEST;
106			break;
107		case 'L':
108			cmd = PIN_LONGTERM_BENCHMARK;
109			break;
110		case 'c':
111			cmd = DUMP_USER_PAGES_TEST;
112			/*
113			 * Dump page 0 (index 1). May be overridden later, by
114			 * user's non-option arguments.
115			 *
116			 * .which_pages is zero-based, so that zero can mean "do
117			 * nothing".
118			 */
119			gup.which_pages[0] = 1;
120			break;
121		case 'p':
122			/* works only with DUMP_USER_PAGES_TEST */
123			gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
124			break;
125		case 'F':
126			/* strtol, so you can pass flags in hex form */
127			gup.gup_flags = strtol(optarg, 0, 0);
128			break;
129		case 'j':
130			nthreads = atoi(optarg);
131			break;
132		case 'm':
133			size = atoi(optarg) * MB;
134			break;
135		case 'r':
136			repeats = atoi(optarg);
137			break;
138		case 'n':
139			nr_pages = atoi(optarg);
140			break;
141		case 't':
142			thp = 1;
143			break;
144		case 'T':
145			thp = 0;
146			break;
147		case 'U':
148			cmd = GUP_BASIC_TEST;
149			break;
150		case 'u':
151			cmd = GUP_FAST_BENCHMARK;
152			break;
153		case 'w':
154			write = 1;
155			break;
156		case 'W':
157			write = 0;
158			break;
159		case 'f':
160			file = optarg;
161			break;
162		case 'S':
163			flags &= ~MAP_PRIVATE;
164			flags |= MAP_SHARED;
165			break;
166		case 'H':
167			flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
168			break;
169		case 'z':
170			/* fault pages in gup, do not fault in userland */
171			touch = 1;
172			break;
173		default:
174			return -1;
175		}
176	}
177
178	if (optind < argc) {
179		int extra_arg_count = 0;
180		/*
181		 * For example:
182		 *
183		 *   ./gup_test -c 0 1 0x1001
184		 *
185		 * ...to dump pages 0, 1, and 4097
186		 */
187
188		while ((optind < argc) &&
189		       (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
190			/*
191			 * Do the 1-based indexing here, so that the user can
192			 * use normal 0-based indexing on the command line.
193			 */
194			long page_index = strtol(argv[optind], 0, 0) + 1;
195
196			gup.which_pages[extra_arg_count] = page_index;
197			extra_arg_count++;
198			optind++;
199		}
200	}
201
202	filed = open(file, O_RDWR|O_CREAT);
203	if (filed < 0) {
204		perror("open");
205		exit(filed);
206	}
207
208	gup.nr_pages_per_call = nr_pages;
209	if (write)
210		gup.gup_flags |= FOLL_WRITE;
211
212	gup_fd = open(GUP_TEST_FILE, O_RDWR);
213	if (gup_fd == -1) {
214		switch (errno) {
215		case EACCES:
216			if (getuid())
217				printf("Please run this test as root\n");
218			break;
219		case ENOENT:
220			if (opendir("/sys/kernel/debug") == NULL) {
221				printf("mount debugfs at /sys/kernel/debug\n");
222				break;
223			}
224			printf("check if CONFIG_GUP_TEST is enabled in kernel config\n");
225			break;
226		default:
227			perror("failed to open " GUP_TEST_FILE);
228			break;
229		}
230		exit(KSFT_SKIP);
231	}
232
233	p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
234	if (p == MAP_FAILED) {
235		perror("mmap");
236		exit(1);
237	}
238	gup.addr = (unsigned long)p;
239
240	if (thp == 1)
241		madvise(p, size, MADV_HUGEPAGE);
242	else if (thp == 0)
243		madvise(p, size, MADV_NOHUGEPAGE);
244
245	/*
246	 * FOLL_TOUCH, in gup_test, is used as an either/or case: either
247	 * fault pages in from the kernel via FOLL_TOUCH, or fault them
248	 * in here, from user space. This allows comparison of performance
249	 * between those two cases.
250	 */
251	if (touch) {
252		gup.gup_flags |= FOLL_TOUCH;
253	} else {
254		for (; (unsigned long)p < gup.addr + size; p += PAGE_SIZE)
255			p[0] = 0;
256	}
257
258	tid = malloc(sizeof(pthread_t) * nthreads);
259	assert(tid);
260	for (i = 0; i < nthreads; i++) {
261		ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
262		assert(ret == 0);
263	}
264	for (i = 0; i < nthreads; i++) {
265		ret = pthread_join(tid[i], NULL);
266		assert(ret == 0);
267	}
268	free(tid);
269
270	return 0;
271}