1// SPDX-License-Identifier: GPL-2.0
  2#define _GNU_SOURCE
  3#include <sys/mman.h>
  4#include <stdint.h>
  5#include <unistd.h>
  6#include <string.h>
  7#include <sys/time.h>
  8#include <sys/resource.h>
  9#include <stdbool.h>
 10#include "mlock2.h"
 11
 12#include "../kselftest.h"
 13
 14struct vm_boundaries {
 15	unsigned long start;
 16	unsigned long end;
 17};
 18
 19static int get_vm_area(unsigned long addr, struct vm_boundaries *area)
 20{
 21	FILE *file;
 22	int ret = 1;
 23	char line[1024] = {0};
 24	char *end_addr;
 25	char *stop;
 26	unsigned long start;
 27	unsigned long end;
 28
 29	if (!area)
 30		return ret;
 31
 32	file = fopen("/proc/self/maps", "r");
 33	if (!file) {
 34		perror("fopen");
 35		return ret;
 36	}
 37
 38	memset(area, 0, sizeof(struct vm_boundaries));
 39
 40	while(fgets(line, 1024, file)) {
 41		end_addr = strchr(line, '-');
 42		if (!end_addr) {
 43			printf("cannot parse /proc/self/maps\n");
 44			goto out;
 45		}
 46		*end_addr = '\0';
 47		end_addr++;
 48		stop = strchr(end_addr, ' ');
 49		if (!stop) {
 50			printf("cannot parse /proc/self/maps\n");
 51			goto out;
 52		}
 53		stop = '\0';
 54
 55		sscanf(line, "%lx", &start);
 56		sscanf(end_addr, "%lx", &end);
 57
 58		if (start <= addr && end > addr) {
 59			area->start = start;
 60			area->end = end;
 61			ret = 0;
 62			goto out;
 63		}
 64	}
 65out:
 66	fclose(file);
 67	return ret;
 68}
 69
 70static uint64_t get_pageflags(unsigned long addr)
 71{
 72	FILE *file;
 73	uint64_t pfn;
 74	unsigned long offset;
 75
 76	file = fopen("/proc/self/pagemap", "r");
 77	if (!file) {
 78		perror("fopen pagemap");
 79		_exit(1);
 80	}
 81
 82	offset = addr / getpagesize() * sizeof(pfn);
 83
 84	if (fseek(file, offset, SEEK_SET)) {
 85		perror("fseek pagemap");
 86		_exit(1);
 87	}
 88
 89	if (fread(&pfn, sizeof(pfn), 1, file) != 1) {
 90		perror("fread pagemap");
 91		_exit(1);
 92	}
 93
 94	fclose(file);
 95	return pfn;
 96}
 97
 98static uint64_t get_kpageflags(unsigned long pfn)
 99{
100	uint64_t flags;
101	FILE *file;
102
103	file = fopen("/proc/kpageflags", "r");
104	if (!file) {
105		perror("fopen kpageflags");
106		_exit(1);
107	}
108
109	if (fseek(file, pfn * sizeof(flags), SEEK_SET)) {
110		perror("fseek kpageflags");
111		_exit(1);
112	}
113
114	if (fread(&flags, sizeof(flags), 1, file) != 1) {
115		perror("fread kpageflags");
116		_exit(1);
117	}
118
119	fclose(file);
120	return flags;
121}
122
123#define VMFLAGS "VmFlags:"
124
125static bool is_vmflag_set(unsigned long addr, const char *vmflag)
126{
127	char *line = NULL;
128	char *flags;
129	size_t size = 0;
130	bool ret = false;
131	FILE *smaps;
132
133	smaps = seek_to_smaps_entry(addr);
134	if (!smaps) {
135		printf("Unable to parse /proc/self/smaps\n");
136		goto out;
137	}
138
139	while (getline(&line, &size, smaps) > 0) {
140		if (!strstr(line, VMFLAGS)) {
141			free(line);
142			line = NULL;
143			size = 0;
144			continue;
145		}
146
147		flags = line + strlen(VMFLAGS);
148		ret = (strstr(flags, vmflag) != NULL);
149		goto out;
150	}
151
152out:
153	free(line);
154	fclose(smaps);
155	return ret;
156}
157
158#define SIZE "Size:"
159#define RSS  "Rss:"
160#define LOCKED "lo"
161
162static bool is_vma_lock_on_fault(unsigned long addr)
163{
164	bool ret = false;
165	bool locked;
166	FILE *smaps = NULL;
167	unsigned long vma_size, vma_rss;
168	char *line = NULL;
169	char *value;
170	size_t size = 0;
171
172	locked = is_vmflag_set(addr, LOCKED);
173	if (!locked)
174		goto out;
175
176	smaps = seek_to_smaps_entry(addr);
177	if (!smaps) {
178		printf("Unable to parse /proc/self/smaps\n");
179		goto out;
180	}
181
182	while (getline(&line, &size, smaps) > 0) {
183		if (!strstr(line, SIZE)) {
184			free(line);
185			line = NULL;
186			size = 0;
187			continue;
188		}
189
190		value = line + strlen(SIZE);
191		if (sscanf(value, "%lu kB", &vma_size) < 1) {
192			printf("Unable to parse smaps entry for Size\n");
193			goto out;
194		}
195		break;
196	}
197
198	while (getline(&line, &size, smaps) > 0) {
199		if (!strstr(line, RSS)) {
200			free(line);
201			line = NULL;
202			size = 0;
203			continue;
204		}
205
206		value = line + strlen(RSS);
207		if (sscanf(value, "%lu kB", &vma_rss) < 1) {
208			printf("Unable to parse smaps entry for Rss\n");
209			goto out;
210		}
211		break;
212	}
213
214	ret = locked && (vma_rss < vma_size);
215out:
216	free(line);
217	if (smaps)
218		fclose(smaps);
219	return ret;
220}
221
222#define PRESENT_BIT     0x8000000000000000ULL
223#define PFN_MASK        0x007FFFFFFFFFFFFFULL
224#define UNEVICTABLE_BIT (1UL << 18)
225
226static int lock_check(char *map)
227{
228	unsigned long page_size = getpagesize();
229	uint64_t page1_flags, page2_flags;
230
231	page1_flags = get_pageflags((unsigned long)map);
232	page2_flags = get_pageflags((unsigned long)map + page_size);
233
234	/* Both pages should be present */
235	if (((page1_flags & PRESENT_BIT) == 0) ||
236	    ((page2_flags & PRESENT_BIT) == 0)) {
237		printf("Failed to make both pages present\n");
238		return 1;
239	}
240
241	page1_flags = get_kpageflags(page1_flags & PFN_MASK);
242	page2_flags = get_kpageflags(page2_flags & PFN_MASK);
243
244	/* Both pages should be unevictable */
245	if (((page1_flags & UNEVICTABLE_BIT) == 0) ||
246	    ((page2_flags & UNEVICTABLE_BIT) == 0)) {
247		printf("Failed to make both pages unevictable\n");
248		return 1;
249	}
250
251	if (!is_vmflag_set((unsigned long)map, LOCKED)) {
252		printf("VMA flag %s is missing on page 1\n", LOCKED);
253		return 1;
254	}
255
256	if (!is_vmflag_set((unsigned long)map + page_size, LOCKED)) {
257		printf("VMA flag %s is missing on page 2\n", LOCKED);
258		return 1;
259	}
260
261	return 0;
262}
263
264static int unlock_lock_check(char *map)
265{
266	unsigned long page_size = getpagesize();
267	uint64_t page1_flags, page2_flags;
268
269	page1_flags = get_pageflags((unsigned long)map);
270	page2_flags = get_pageflags((unsigned long)map + page_size);
271	page1_flags = get_kpageflags(page1_flags & PFN_MASK);
272	page2_flags = get_kpageflags(page2_flags & PFN_MASK);
273
274	if ((page1_flags & UNEVICTABLE_BIT) || (page2_flags & UNEVICTABLE_BIT)) {
275		printf("A page is still marked unevictable after unlock\n");
276		return 1;
277	}
278
279	if (is_vmflag_set((unsigned long)map, LOCKED)) {
280		printf("VMA flag %s is present on page 1 after unlock\n", LOCKED);
281		return 1;
282	}
283
284	if (is_vmflag_set((unsigned long)map + page_size, LOCKED)) {
285		printf("VMA flag %s is present on page 2 after unlock\n", LOCKED);
286		return 1;
287	}
288
289	return 0;
290}
291
292static int test_mlock_lock()
293{
294	char *map;
295	int ret = 1;
296	unsigned long page_size = getpagesize();
297
298	map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE,
299		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
300	if (map == MAP_FAILED) {
301		perror("test_mlock_locked mmap");
302		goto out;
303	}
304
305	if (mlock2_(map, 2 * page_size, 0)) {
306		if (errno == ENOSYS) {
307			printf("Cannot call new mlock family, skipping test\n");
308			_exit(KSFT_SKIP);
309		}
310		perror("mlock2(0)");
311		goto unmap;
312	}
313
314	if (lock_check(map))
315		goto unmap;
316
317	/* Now unlock and recheck attributes */
318	if (munlock(map, 2 * page_size)) {
319		perror("munlock()");
320		goto unmap;
321	}
322
323	ret = unlock_lock_check(map);
324
325unmap:
326	munmap(map, 2 * page_size);
327out:
328	return ret;
329}
330
331static int onfault_check(char *map)
332{
333	unsigned long page_size = getpagesize();
334	uint64_t page1_flags, page2_flags;
335
336	page1_flags = get_pageflags((unsigned long)map);
337	page2_flags = get_pageflags((unsigned long)map + page_size);
338
339	/* Neither page should be present */
340	if ((page1_flags & PRESENT_BIT) || (page2_flags & PRESENT_BIT)) {
341		printf("Pages were made present by MLOCK_ONFAULT\n");
342		return 1;
343	}
344
345	*map = 'a';
346	page1_flags = get_pageflags((unsigned long)map);
347	page2_flags = get_pageflags((unsigned long)map + page_size);
348
349	/* Only page 1 should be present */
350	if ((page1_flags & PRESENT_BIT) == 0) {
351		printf("Page 1 is not present after fault\n");
352		return 1;
353	} else if (page2_flags & PRESENT_BIT) {
354		printf("Page 2 was made present\n");
355		return 1;
356	}
357
358	page1_flags = get_kpageflags(page1_flags & PFN_MASK);
359
360	/* Page 1 should be unevictable */
361	if ((page1_flags & UNEVICTABLE_BIT) == 0) {
362		printf("Failed to make faulted page unevictable\n");
363		return 1;
364	}
365
366	if (!is_vma_lock_on_fault((unsigned long)map)) {
367		printf("VMA is not marked for lock on fault\n");
368		return 1;
369	}
370
371	if (!is_vma_lock_on_fault((unsigned long)map + page_size)) {
372		printf("VMA is not marked for lock on fault\n");
373		return 1;
374	}
375
376	return 0;
377}
378
379static int unlock_onfault_check(char *map)
380{
381	unsigned long page_size = getpagesize();
382	uint64_t page1_flags;
383
384	page1_flags = get_pageflags((unsigned long)map);
385	page1_flags = get_kpageflags(page1_flags & PFN_MASK);
386
387	if (page1_flags & UNEVICTABLE_BIT) {
388		printf("Page 1 is still marked unevictable after unlock\n");
389		return 1;
390	}
391
392	if (is_vma_lock_on_fault((unsigned long)map) ||
393	    is_vma_lock_on_fault((unsigned long)map + page_size)) {
394		printf("VMA is still lock on fault after unlock\n");
395		return 1;
396	}
397
398	return 0;
399}
400
401static int test_mlock_onfault()
402{
403	char *map;
404	int ret = 1;
405	unsigned long page_size = getpagesize();
406
407	map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE,
408		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
409	if (map == MAP_FAILED) {
410		perror("test_mlock_locked mmap");
411		goto out;
412	}
413
414	if (mlock2_(map, 2 * page_size, MLOCK_ONFAULT)) {
415		if (errno == ENOSYS) {
416			printf("Cannot call new mlock family, skipping test\n");
417			_exit(KSFT_SKIP);
418		}
419		perror("mlock2(MLOCK_ONFAULT)");
420		goto unmap;
421	}
422
423	if (onfault_check(map))
424		goto unmap;
425
426	/* Now unlock and recheck attributes */
427	if (munlock(map, 2 * page_size)) {
428		if (errno == ENOSYS) {
429			printf("Cannot call new mlock family, skipping test\n");
430			_exit(KSFT_SKIP);
431		}
432		perror("munlock()");
433		goto unmap;
434	}
435
436	ret = unlock_onfault_check(map);
437unmap:
438	munmap(map, 2 * page_size);
439out:
440	return ret;
441}
442
443static int test_lock_onfault_of_present()
444{
445	char *map;
446	int ret = 1;
447	unsigned long page_size = getpagesize();
448	uint64_t page1_flags, page2_flags;
449
450	map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE,
451		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
452	if (map == MAP_FAILED) {
453		perror("test_mlock_locked mmap");
454		goto out;
455	}
456
457	*map = 'a';
458
459	if (mlock2_(map, 2 * page_size, MLOCK_ONFAULT)) {
460		if (errno == ENOSYS) {
461			printf("Cannot call new mlock family, skipping test\n");
462			_exit(KSFT_SKIP);
463		}
464		perror("mlock2(MLOCK_ONFAULT)");
465		goto unmap;
466	}
467
468	page1_flags = get_pageflags((unsigned long)map);
469	page2_flags = get_pageflags((unsigned long)map + page_size);
470	page1_flags = get_kpageflags(page1_flags & PFN_MASK);
471	page2_flags = get_kpageflags(page2_flags & PFN_MASK);
472
473	/* Page 1 should be unevictable */
474	if ((page1_flags & UNEVICTABLE_BIT) == 0) {
475		printf("Failed to make present page unevictable\n");
476		goto unmap;
477	}
478
479	if (!is_vma_lock_on_fault((unsigned long)map) ||
480	    !is_vma_lock_on_fault((unsigned long)map + page_size)) {
481		printf("VMA with present pages is not marked lock on fault\n");
482		goto unmap;
483	}
484	ret = 0;
485unmap:
486	munmap(map, 2 * page_size);
487out:
488	return ret;
489}
490
491static int test_munlockall()
492{
493	char *map;
494	int ret = 1;
495	unsigned long page_size = getpagesize();
496
497	map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE,
498		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
499
500	if (map == MAP_FAILED) {
501		perror("test_munlockall mmap");
502		goto out;
503	}
504
505	if (mlockall(MCL_CURRENT)) {
506		perror("mlockall(MCL_CURRENT)");
507		goto out;
508	}
509
510	if (lock_check(map))
511		goto unmap;
512
513	if (munlockall()) {
514		perror("munlockall()");
515		goto unmap;
516	}
517
518	if (unlock_lock_check(map))
519		goto unmap;
520
521	munmap(map, 2 * page_size);
522
523	map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE,
524		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
525
526	if (map == MAP_FAILED) {
527		perror("test_munlockall second mmap");
528		goto out;
529	}
530
531	if (mlockall(MCL_CURRENT | MCL_ONFAULT)) {
532		perror("mlockall(MCL_CURRENT | MCL_ONFAULT)");
533		goto unmap;
534	}
535
536	if (onfault_check(map))
537		goto unmap;
538
539	if (munlockall()) {
540		perror("munlockall()");
541		goto unmap;
542	}
543
544	if (unlock_onfault_check(map))
545		goto unmap;
546
547	if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
548		perror("mlockall(MCL_CURRENT | MCL_FUTURE)");
549		goto out;
550	}
551
552	if (lock_check(map))
553		goto unmap;
554
555	if (munlockall()) {
556		perror("munlockall()");
557		goto unmap;
558	}
559
560	ret = unlock_lock_check(map);
561
562unmap:
563	munmap(map, 2 * page_size);
564out:
565	munlockall();
566	return ret;
567}
568
569static int test_vma_management(bool call_mlock)
570{
571	int ret = 1;
572	void *map;
573	unsigned long page_size = getpagesize();
574	struct vm_boundaries page1;
575	struct vm_boundaries page2;
576	struct vm_boundaries page3;
577
578	map = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
579		   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
580	if (map == MAP_FAILED) {
581		perror("mmap()");
582		return ret;
583	}
584
585	if (call_mlock && mlock2_(map, 3 * page_size, MLOCK_ONFAULT)) {
586		if (errno == ENOSYS) {
587			printf("Cannot call new mlock family, skipping test\n");
588			_exit(KSFT_SKIP);
589		}
590		perror("mlock(ONFAULT)\n");
591		goto out;
592	}
593
594	if (get_vm_area((unsigned long)map, &page1) ||
595	    get_vm_area((unsigned long)map + page_size, &page2) ||
596	    get_vm_area((unsigned long)map + page_size * 2, &page3)) {
597		printf("couldn't find mapping in /proc/self/maps\n");
598		goto out;
599	}
600
601	/*
602	 * Before we unlock a portion, we need to that all three pages are in
603	 * the same VMA.  If they are not we abort this test (Note that this is
604	 * not a failure)
605	 */
606	if (page1.start != page2.start || page2.start != page3.start) {
607		printf("VMAs are not merged to start, aborting test\n");
608		ret = 0;
609		goto out;
610	}
611
612	if (munlock(map + page_size, page_size)) {
613		perror("munlock()");
614		goto out;
615	}
616
617	if (get_vm_area((unsigned long)map, &page1) ||
618	    get_vm_area((unsigned long)map + page_size, &page2) ||
619	    get_vm_area((unsigned long)map + page_size * 2, &page3)) {
620		printf("couldn't find mapping in /proc/self/maps\n");
621		goto out;
622	}
623
624	/* All three VMAs should be different */
625	if (page1.start == page2.start || page2.start == page3.start) {
626		printf("failed to split VMA for munlock\n");
627		goto out;
628	}
629
630	/* Now unlock the first and third page and check the VMAs again */
631	if (munlock(map, page_size * 3)) {
632		perror("munlock()");
633		goto out;
634	}
635
636	if (get_vm_area((unsigned long)map, &page1) ||
637	    get_vm_area((unsigned long)map + page_size, &page2) ||
638	    get_vm_area((unsigned long)map + page_size * 2, &page3)) {
639		printf("couldn't find mapping in /proc/self/maps\n");
640		goto out;
641	}
642
643	/* Now all three VMAs should be the same */
644	if (page1.start != page2.start || page2.start != page3.start) {
645		printf("failed to merge VMAs after munlock\n");
646		goto out;
647	}
648
649	ret = 0;
650out:
651	munmap(map, 3 * page_size);
652	return ret;
653}
654
655static int test_mlockall(int (test_function)(bool call_mlock))
656{
657	int ret = 1;
658
659	if (mlockall(MCL_CURRENT | MCL_ONFAULT | MCL_FUTURE)) {
660		perror("mlockall");
661		return ret;
662	}
663
664	ret = test_function(false);
665	munlockall();
666	return ret;
667}
668
669int main(int argc, char **argv)
670{
671	int ret = 0;
672	ret += test_mlock_lock();
673	ret += test_mlock_onfault();
674	ret += test_munlockall();
675	ret += test_lock_onfault_of_present();
676	ret += test_vma_management(true);
677	ret += test_mlockall(test_vma_management);
678	return ret;
679}