1/*
  2 * Stress test for transparent huge pages, memory compaction and migration.
  3 *
  4 * Authors: Konstantin Khlebnikov <koct9i@gmail.com>
  5 *
  6 * This is free and unencumbered software released into the public domain.
  7 */
  8
  9#include <stdlib.h>
 10#include <stdio.h>
 11#include <stdint.h>
 12#include <err.h>
 13#include <time.h>
 14#include <unistd.h>
 15#include <fcntl.h>
 16#include <string.h>
 17#include <sys/mman.h>
 18
 19#define PAGE_SHIFT 12
 20#define HPAGE_SHIFT 21
 21
 22#define PAGE_SIZE (1 << PAGE_SHIFT)
 23#define HPAGE_SIZE (1 << HPAGE_SHIFT)
 24
 25#define PAGEMAP_PRESENT(ent)	(((ent) & (1ull << 63)) != 0)
 26#define PAGEMAP_PFN(ent)	((ent) & ((1ull << 55) - 1))
 27
 28int pagemap_fd;
 29
 30int64_t allocate_transhuge(void *ptr)
 31{
 32	uint64_t ent[2];
 33
 34	/* drop pmd */
 35	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
 36				MAP_FIXED | MAP_ANONYMOUS |
 37				MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
 38		errx(2, "mmap transhuge");
 39
 40	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
 41		err(2, "MADV_HUGEPAGE");
 42
 43	/* allocate transparent huge page */
 44	*(volatile void **)ptr = ptr;
 45
 46	if (pread(pagemap_fd, ent, sizeof(ent),
 47			(uintptr_t)ptr >> (PAGE_SHIFT - 3)) != sizeof(ent))
 48		err(2, "read pagemap");
 49
 50	if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
 51	    PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
 52	    !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - PAGE_SHIFT)) - 1)))
 53		return PAGEMAP_PFN(ent[0]);
 54
 55	return -1;
 56}
 57
 58int main(int argc, char **argv)
 59{
 60	size_t ram, len;
 61	void *ptr, *p;
 62	struct timespec a, b;
 63	double s;
 64	uint8_t *map;
 65	size_t map_len;
 66
 67	ram = sysconf(_SC_PHYS_PAGES);
 68	if (ram > SIZE_MAX / sysconf(_SC_PAGESIZE) / 4)
 69		ram = SIZE_MAX / 4;
 70	else
 71		ram *= sysconf(_SC_PAGESIZE);
 72
 73	if (argc == 1)
 74		len = ram;
 75	else if (!strcmp(argv[1], "-h"))
 76		errx(1, "usage: %s [size in MiB]", argv[0]);
 77	else
 78		len = atoll(argv[1]) << 20;
 79
 80	warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
 81	      " and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
 82	      len >> (20 + HPAGE_SHIFT - PAGE_SHIFT - 1));
 83
 84	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
 85	if (pagemap_fd < 0)
 86		err(2, "open pagemap");
 87
 88	len -= len % HPAGE_SIZE;
 89	ptr = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
 90			MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
 91	if (ptr == MAP_FAILED)
 92		err(2, "initial mmap");
 93	ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;
 94
 95	if (madvise(ptr, len, MADV_HUGEPAGE))
 96		err(2, "MADV_HUGEPAGE");
 97
 98	map_len = ram >> (HPAGE_SHIFT - 1);
 99	map = malloc(map_len);
100	if (!map)
101		errx(2, "map malloc");
102
103	while (1) {
104		int nr_succeed = 0, nr_failed = 0, nr_pages = 0;
105
106		memset(map, 0, map_len);
107
108		clock_gettime(CLOCK_MONOTONIC, &a);
109		for (p = ptr; p < ptr + len; p += HPAGE_SIZE) {
110			int64_t pfn;
111
112			pfn = allocate_transhuge(p);
113
114			if (pfn < 0) {
115				nr_failed++;
116			} else {
117				size_t idx = pfn >> (HPAGE_SHIFT - PAGE_SHIFT);
118
119				nr_succeed++;
120				if (idx >= map_len) {
121					map = realloc(map, idx + 1);
122					if (!map)
123						errx(2, "map realloc");
124					memset(map + map_len, 0, idx + 1 - map_len);
125					map_len = idx + 1;
126				}
127				if (!map[idx])
128					nr_pages++;
129				map[idx] = 1;
130			}
131
132			/* split transhuge page, keep last page */
133			if (madvise(p, HPAGE_SIZE - PAGE_SIZE, MADV_DONTNEED))
134				err(2, "MADV_DONTNEED");
135		}
136		clock_gettime(CLOCK_MONOTONIC, &b);
137		s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.;
138
139		warnx("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t"
140		      "%4d succeed, %4d failed, %4d different pages",
141		      s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20),
142		      nr_succeed, nr_failed, nr_pages);
143	}
144}