1// SPDX-License-Identifier: GPL-2.0
  2#include <string.h>
  3#include <fcntl.h>
  4#include <dirent.h>
  5#include <sys/ioctl.h>
  6#include <linux/userfaultfd.h>
  7#include <linux/fs.h>
  8#include <sys/syscall.h>
  9#include <unistd.h>
 10#include "../kselftest.h"
 11#include "vm_util.h"
 12
 13#define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
 14#define SMAP_FILE_PATH "/proc/self/smaps"
 15#define STATUS_FILE_PATH "/proc/self/status"
 16#define MAX_LINE_LENGTH 500
 17
 18unsigned int __page_size;
 19unsigned int __page_shift;
 20
 21uint64_t pagemap_get_entry(int fd, char *start)
 22{
 23	const unsigned long pfn = (unsigned long)start / getpagesize();
 24	uint64_t entry;
 25	int ret;
 26
 27	ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
 28	if (ret != sizeof(entry))
 29		ksft_exit_fail_msg("reading pagemap failed\n");
 30	return entry;
 31}
 32
 33static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
 34{
 35	struct pm_scan_arg arg;
 36
 37	arg.start = (uintptr_t)start;
 38	arg.end = (uintptr_t)(start + psize());
 39	arg.vec = (uintptr_t)r;
 40	arg.vec_len = 1;
 41	arg.flags = 0;
 42	arg.size = sizeof(struct pm_scan_arg);
 43	arg.max_pages = 0;
 44	arg.category_inverted = 0;
 45	arg.category_mask = 0;
 46	arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
 47				  PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
 48				  PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
 49	arg.return_mask = arg.category_anyof_mask;
 50
 51	return ioctl(fd, PAGEMAP_SCAN, &arg);
 52}
 53
 54static uint64_t pagemap_scan_get_categories(int fd, char *start)
 55{
 56	struct page_region r;
 57	long ret;
 58
 59	ret = __pagemap_scan_get_categories(fd, start, &r);
 60	if (ret < 0)
 61		ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
 62	if (ret == 0)
 63		return 0;
 64	return r.categories;
 65}
 66
 67/* `start` is any valid address. */
 68static bool pagemap_scan_supported(int fd, char *start)
 69{
 70	static int supported = -1;
 71	int ret;
 72
 73	if (supported != -1)
 74		return supported;
 75
 76	/* Provide an invalid address in order to trigger EFAULT. */
 77	ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
 78	if (ret == 0)
 79		ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
 80
 81	supported = errno == EFAULT;
 82
 83	return supported;
 84}
 85
 86static bool page_entry_is(int fd, char *start, char *desc,
 87			  uint64_t pagemap_flags, uint64_t pagescan_flags)
 88{
 89	bool m = pagemap_get_entry(fd, start) & pagemap_flags;
 90
 91	if (pagemap_scan_supported(fd, start)) {
 92		bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
 93
 94		if (m == s)
 95			return m;
 96
 97		ksft_exit_fail_msg(
 98			"read and ioctl return unmatched results for %s: %d %d", desc, m, s);
 99	}
100	return m;
101}
102
103bool pagemap_is_softdirty(int fd, char *start)
104{
105	return page_entry_is(fd, start, "soft-dirty",
106				PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
107}
108
109bool pagemap_is_swapped(int fd, char *start)
110{
111	return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
112}
113
114bool pagemap_is_populated(int fd, char *start)
115{
116	return page_entry_is(fd, start, "populated",
117				PM_PRESENT | PM_SWAP,
118				PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
119}
120
121unsigned long pagemap_get_pfn(int fd, char *start)
122{
123	uint64_t entry = pagemap_get_entry(fd, start);
124
125	/* If present (63th bit), PFN is at bit 0 -- 54. */
126	if (entry & PM_PRESENT)
127		return entry & 0x007fffffffffffffull;
128	return -1ul;
129}
130
131void clear_softdirty(void)
132{
133	int ret;
134	const char *ctrl = "4";
135	int fd = open("/proc/self/clear_refs", O_WRONLY);
136
137	if (fd < 0)
138		ksft_exit_fail_msg("opening clear_refs failed\n");
139	ret = write(fd, ctrl, strlen(ctrl));
140	close(fd);
141	if (ret != strlen(ctrl))
142		ksft_exit_fail_msg("writing clear_refs failed\n");
143}
144
145bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
146{
147	while (fgets(buf, len, fp)) {
148		if (!strncmp(buf, pattern, strlen(pattern)))
149			return true;
150	}
151	return false;
152}
153
154uint64_t read_pmd_pagesize(void)
155{
156	int fd;
157	char buf[20];
158	ssize_t num_read;
159
160	fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
161	if (fd == -1)
162		return 0;
163
164	num_read = read(fd, buf, 19);
165	if (num_read < 1) {
166		close(fd);
167		return 0;
168	}
169	buf[num_read] = '\0';
170	close(fd);
171
172	return strtoul(buf, NULL, 10);
173}
174
175unsigned long rss_anon(void)
176{
177	unsigned long rss_anon = 0;
178	FILE *fp;
179	char buffer[MAX_LINE_LENGTH];
180
181	fp = fopen(STATUS_FILE_PATH, "r");
182	if (!fp)
183		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
184
185	if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
186		goto err_out;
187
188	if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
189		ksft_exit_fail_msg("Reading status error\n");
190
191err_out:
192	fclose(fp);
193	return rss_anon;
194}
195
196bool __check_huge(void *addr, char *pattern, int nr_hpages,
197		  uint64_t hpage_size)
198{
199	uint64_t thp = -1;
200	int ret;
201	FILE *fp;
202	char buffer[MAX_LINE_LENGTH];
203	char addr_pattern[MAX_LINE_LENGTH];
204
205	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
206		       (unsigned long) addr);
207	if (ret >= MAX_LINE_LENGTH)
208		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
209
210	fp = fopen(SMAP_FILE_PATH, "r");
211	if (!fp)
212		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
213
214	if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
215		goto err_out;
216
217	/*
218	 * Fetch the pattern in the same block and check the number of
219	 * hugepages.
220	 */
221	if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
222		goto err_out;
223
224	snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
225
226	if (sscanf(buffer, addr_pattern, &thp) != 1)
227		ksft_exit_fail_msg("Reading smap error\n");
228
229err_out:
230	fclose(fp);
231	return thp == (nr_hpages * (hpage_size >> 10));
232}
233
234bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
235{
236	return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
237}
238
239bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
240{
241	return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
242}
243
244bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
245{
246	return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
247}
248
249int64_t allocate_transhuge(void *ptr, int pagemap_fd)
250{
251	uint64_t ent[2];
252
253	/* drop pmd */
254	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
255		 MAP_FIXED | MAP_ANONYMOUS |
256		 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
257		ksft_exit_fail_msg("mmap transhuge\n");
258
259	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
260		ksft_exit_fail_msg("MADV_HUGEPAGE\n");
261
262	/* allocate transparent huge page */
263	*(volatile void **)ptr = ptr;
264
265	if (pread(pagemap_fd, ent, sizeof(ent),
266		  (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
267		ksft_exit_fail_msg("read pagemap\n");
268
269	if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
270	    PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
271	    !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
272		return PAGEMAP_PFN(ent[0]);
273
274	return -1;
275}
276
277unsigned long default_huge_page_size(void)
278{
279	unsigned long hps = 0;
280	char *line = NULL;
281	size_t linelen = 0;
282	FILE *f = fopen("/proc/meminfo", "r");
283
284	if (!f)
285		return 0;
286	while (getline(&line, &linelen, f) > 0) {
287		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
288			hps <<= 10;
289			break;
290		}
291	}
292
293	free(line);
294	fclose(f);
295	return hps;
296}
297
298int detect_hugetlb_page_sizes(size_t sizes[], int max)
299{
300	DIR *dir = opendir("/sys/kernel/mm/hugepages/");
301	int count = 0;
302
303	if (!dir)
304		return 0;
305
306	while (count < max) {
307		struct dirent *entry = readdir(dir);
308		size_t kb;
309
310		if (!entry)
311			break;
312		if (entry->d_type != DT_DIR)
313			continue;
314		if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
315			continue;
316		sizes[count++] = kb * 1024;
317		ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
318			       kb);
319	}
320	closedir(dir);
321	return count;
322}
323
324/* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
325int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
326			      bool miss, bool wp, bool minor, uint64_t *ioctls)
327{
328	struct uffdio_register uffdio_register = { 0 };
329	uint64_t mode = 0;
330	int ret = 0;
331
332	if (miss)
333		mode |= UFFDIO_REGISTER_MODE_MISSING;
334	if (wp)
335		mode |= UFFDIO_REGISTER_MODE_WP;
336	if (minor)
337		mode |= UFFDIO_REGISTER_MODE_MINOR;
338
339	uffdio_register.range.start = (unsigned long)addr;
340	uffdio_register.range.len = len;
341	uffdio_register.mode = mode;
342
343	if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
344		ret = -errno;
345	else if (ioctls)
346		*ioctls = uffdio_register.ioctls;
347
348	return ret;
349}
350
351int uffd_register(int uffd, void *addr, uint64_t len,
352		  bool miss, bool wp, bool minor)
353{
354	return uffd_register_with_ioctls(uffd, addr, len,
355					 miss, wp, minor, NULL);
356}
357
358int uffd_unregister(int uffd, void *addr, uint64_t len)
359{
360	struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
361	int ret = 0;
362
363	if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
364		ret = -errno;
365
366	return ret;
367}
368
369unsigned long get_free_hugepages(void)
370{
371	unsigned long fhp = 0;
372	char *line = NULL;
373	size_t linelen = 0;
374	FILE *f = fopen("/proc/meminfo", "r");
375
376	if (!f)
377		return fhp;
378	while (getline(&line, &linelen, f) > 0) {
379		if (sscanf(line, "HugePages_Free:      %lu", &fhp) == 1)
380			break;
381	}
382
383	free(line);
384	fclose(f);
385	return fhp;
386}