1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/string.h>
  4#include <linux/mm.h>
  5#include <linux/highmem.h>
  6#include <linux/page_ext.h>
  7#include <linux/poison.h>
  8#include <linux/ratelimit.h>
  9
 
 10static bool want_page_poisoning __read_mostly;
 11
 12static int __init early_page_poison_param(char *buf)
 13{
 14	if (!buf)
 15		return -EINVAL;
 16	return strtobool(buf, &want_page_poisoning);
 
 
 
 
 
 
 17}
 18early_param("page_poison", early_page_poison_param);
 19
 20bool page_poisoning_enabled(void)
 21{
 
 
 
 
 
 
 
 
 
 
 22	/*
 23	 * Assumes that debug_pagealloc_enabled is set before
 24	 * free_all_bootmem.
 25	 * Page poisoning is debug page alloc for some arches. If
 26	 * either of those options are enabled, enable poisoning.
 27	 */
 28	return (want_page_poisoning ||
 29		(!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
 30		debug_pagealloc_enabled()));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 31}
 32
 33static void poison_page(struct page *page)
 34{
 35	void *addr = kmap_atomic(page);
 36
 
 37	memset(addr, PAGE_POISON, PAGE_SIZE);
 38	kunmap_atomic(addr);
 39}
 40
 41static void poison_pages(struct page *page, int n)
 42{
 43	int i;
 44
 45	for (i = 0; i < n; i++)
 46		poison_page(page + i);
 47}
 48
 49static bool single_bit_flip(unsigned char a, unsigned char b)
 50{
 51	unsigned char error = a ^ b;
 52
 53	return error && !(error & (error - 1));
 54}
 55
 56static void check_poison_mem(unsigned char *mem, size_t bytes)
 57{
 58	static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10);
 59	unsigned char *start;
 60	unsigned char *end;
 61
 62	if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY))
 63		return;
 64
 65	start = memchr_inv(mem, PAGE_POISON, bytes);
 66	if (!start)
 67		return;
 68
 69	for (end = mem + bytes - 1; end > start; end--) {
 70		if (*end != PAGE_POISON)
 71			break;
 72	}
 73
 74	if (!__ratelimit(&ratelimit))
 75		return;
 76	else if (start == end && single_bit_flip(*start, PAGE_POISON))
 77		pr_err("pagealloc: single bit error\n");
 78	else
 79		pr_err("pagealloc: memory corruption\n");
 80
 81	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start,
 82			end - start + 1, 1);
 83	dump_stack();
 84}
 85
 86static void unpoison_page(struct page *page)
 87{
 88	void *addr;
 89
 
 
 
 90	addr = kmap_atomic(page);
 91	/*
 92	 * Page poisoning when enabled poisons each and every page
 93	 * that is freed to buddy. Thus no extra check is done to
 94	 * see if a page was posioned.
 95	 */
 96	check_poison_mem(addr, PAGE_SIZE);
 
 97	kunmap_atomic(addr);
 98}
 99
100static void unpoison_pages(struct page *page, int n)
101{
102	int i;
103
104	for (i = 0; i < n; i++)
105		unpoison_page(page + i);
106}
107
108void kernel_poison_pages(struct page *page, int numpages, int enable)
109{
110	if (!page_poisoning_enabled())
111		return;
112
113	if (enable)
114		unpoison_pages(page, numpages);
115	else
116		poison_pages(page, numpages);
117}
118
119#ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC
120void __kernel_map_pages(struct page *page, int numpages, int enable)
121{
122	/* This function does nothing, all work is done via poison pages */
123}
124#endif

 
  1#include <linux/kernel.h>
  2#include <linux/string.h>
  3#include <linux/mm.h>
  4#include <linux/highmem.h>
  5#include <linux/page_ext.h>
  6#include <linux/poison.h>
  7#include <linux/ratelimit.h>
  8
  9static bool __page_poisoning_enabled __read_mostly;
 10static bool want_page_poisoning __read_mostly;
 11
 12static int early_page_poison_param(char *buf)
 13{
 14	if (!buf)
 15		return -EINVAL;
 16
 17	if (strcmp(buf, "on") == 0)
 18		want_page_poisoning = true;
 19	else if (strcmp(buf, "off") == 0)
 20		want_page_poisoning = false;
 21
 22	return 0;
 23}
 24early_param("page_poison", early_page_poison_param);
 25
 26bool page_poisoning_enabled(void)
 27{
 28	return __page_poisoning_enabled;
 29}
 30
 31static bool need_page_poisoning(void)
 32{
 33	return want_page_poisoning;
 34}
 35
 36static void init_page_poisoning(void)
 37{
 38	/*
 39	 * page poisoning is debug page alloc for some arches. If either
 40	 * of those options are enabled, enable poisoning
 
 
 41	 */
 42	if (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC)) {
 43		if (!want_page_poisoning && !debug_pagealloc_enabled())
 44			return;
 45	} else {
 46		if (!want_page_poisoning)
 47			return;
 48	}
 49
 50	__page_poisoning_enabled = true;
 51}
 52
 53struct page_ext_operations page_poisoning_ops = {
 54	.need = need_page_poisoning,
 55	.init = init_page_poisoning,
 56};
 57
 58static inline void set_page_poison(struct page *page)
 59{
 60	struct page_ext *page_ext;
 61
 62	page_ext = lookup_page_ext(page);
 63	__set_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
 64}
 65
 66static inline void clear_page_poison(struct page *page)
 67{
 68	struct page_ext *page_ext;
 69
 70	page_ext = lookup_page_ext(page);
 71	__clear_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
 72}
 73
 74bool page_is_poisoned(struct page *page)
 75{
 76	struct page_ext *page_ext;
 77
 78	page_ext = lookup_page_ext(page);
 79	if (!page_ext)
 80		return false;
 81
 82	return test_bit(PAGE_EXT_DEBUG_POISON, &page_ext->flags);
 83}
 84
 85static void poison_page(struct page *page)
 86{
 87	void *addr = kmap_atomic(page);
 88
 89	set_page_poison(page);
 90	memset(addr, PAGE_POISON, PAGE_SIZE);
 91	kunmap_atomic(addr);
 92}
 93
 94static void poison_pages(struct page *page, int n)
 95{
 96	int i;
 97
 98	for (i = 0; i < n; i++)
 99		poison_page(page + i);
100}
101
102static bool single_bit_flip(unsigned char a, unsigned char b)
103{
104	unsigned char error = a ^ b;
105
106	return error && !(error & (error - 1));
107}
108
109static void check_poison_mem(unsigned char *mem, size_t bytes)
110{
111	static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10);
112	unsigned char *start;
113	unsigned char *end;
114
115	if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY))
116		return;
117
118	start = memchr_inv(mem, PAGE_POISON, bytes);
119	if (!start)
120		return;
121
122	for (end = mem + bytes - 1; end > start; end--) {
123		if (*end != PAGE_POISON)
124			break;
125	}
126
127	if (!__ratelimit(&ratelimit))
128		return;
129	else if (start == end && single_bit_flip(*start, PAGE_POISON))
130		pr_err("pagealloc: single bit error\n");
131	else
132		pr_err("pagealloc: memory corruption\n");
133
134	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start,
135			end - start + 1, 1);
136	dump_stack();
137}
138
139static void unpoison_page(struct page *page)
140{
141	void *addr;
142
143	if (!page_is_poisoned(page))
144		return;
145
146	addr = kmap_atomic(page);
 
 
 
 
 
147	check_poison_mem(addr, PAGE_SIZE);
148	clear_page_poison(page);
149	kunmap_atomic(addr);
150}
151
152static void unpoison_pages(struct page *page, int n)
153{
154	int i;
155
156	for (i = 0; i < n; i++)
157		unpoison_page(page + i);
158}
159
160void kernel_poison_pages(struct page *page, int numpages, int enable)
161{
162	if (!page_poisoning_enabled())
163		return;
164
165	if (enable)
166		unpoison_pages(page, numpages);
167	else
168		poison_pages(page, numpages);
169}
170
171#ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC
172void __kernel_map_pages(struct page *page, int numpages, int enable)
173{
174	/* This function does nothing, all work is done via poison pages */
175}
176#endif