1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright (c) 2021, Google LLC.
  5 * Pasha Tatashin <pasha.tatashin@soleen.com>
  6 */
  7#include <linux/kstrtox.h>
  8#include <linux/mm.h>
  9#include <linux/page_table_check.h>
 10#include <linux/swap.h>
 11#include <linux/swapops.h>
 12
 13#undef pr_fmt
 14#define pr_fmt(fmt)	"page_table_check: " fmt
 15
 16struct page_table_check {
 17	atomic_t anon_map_count;
 18	atomic_t file_map_count;
 19};
 20
 21static bool __page_table_check_enabled __initdata =
 22				IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
 23
 24DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
 25EXPORT_SYMBOL(page_table_check_disabled);
 26
 27static int __init early_page_table_check_param(char *buf)
 28{
 29	return kstrtobool(buf, &__page_table_check_enabled);
 30}
 31
 32early_param("page_table_check", early_page_table_check_param);
 33
 34static bool __init need_page_table_check(void)
 35{
 36	return __page_table_check_enabled;
 37}
 38
 39static void __init init_page_table_check(void)
 40{
 41	if (!__page_table_check_enabled)
 42		return;
 43	static_branch_disable(&page_table_check_disabled);
 44}
 45
 46struct page_ext_operations page_table_check_ops = {
 47	.size = sizeof(struct page_table_check),
 48	.need = need_page_table_check,
 49	.init = init_page_table_check,
 50	.need_shared_flags = false,
 51};
 52
 53static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
 54{
 55	BUG_ON(!page_ext);
 56	return page_ext_data(page_ext, &page_table_check_ops);
 57}
 58
 59/*
 60 * An entry is removed from the page table, decrement the counters for that page
 61 * verify that it is of correct type and counters do not become negative.
 62 */
 63static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
 64{
 65	struct page_ext *page_ext;
 66	struct page *page;
 67	unsigned long i;
 68	bool anon;
 69
 70	if (!pfn_valid(pfn))
 71		return;
 72
 73	page = pfn_to_page(pfn);
 74	page_ext = page_ext_get(page);
 75
 76	if (!page_ext)
 77		return;
 78
 79	BUG_ON(PageSlab(page));
 80	anon = PageAnon(page);
 81
 82	for (i = 0; i < pgcnt; i++) {
 83		struct page_table_check *ptc = get_page_table_check(page_ext);
 84
 85		if (anon) {
 86			BUG_ON(atomic_read(&ptc->file_map_count));
 87			BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
 88		} else {
 89			BUG_ON(atomic_read(&ptc->anon_map_count));
 90			BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
 91		}
 92		page_ext = page_ext_next(page_ext);
 93	}
 94	page_ext_put(page_ext);
 95}
 96
 97/*
 98 * A new entry is added to the page table, increment the counters for that page
 99 * verify that it is of correct type and is not being mapped with a different
100 * type to a different process.
101 */
102static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
103				 bool rw)
104{
105	struct page_ext *page_ext;
106	struct page *page;
107	unsigned long i;
108	bool anon;
109
110	if (!pfn_valid(pfn))
111		return;
112
113	page = pfn_to_page(pfn);
114	page_ext = page_ext_get(page);
115
116	if (!page_ext)
117		return;
118
119	BUG_ON(PageSlab(page));
120	anon = PageAnon(page);
121
122	for (i = 0; i < pgcnt; i++) {
123		struct page_table_check *ptc = get_page_table_check(page_ext);
124
125		if (anon) {
126			BUG_ON(atomic_read(&ptc->file_map_count));
127			BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
128		} else {
129			BUG_ON(atomic_read(&ptc->anon_map_count));
130			BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
131		}
132		page_ext = page_ext_next(page_ext);
133	}
134	page_ext_put(page_ext);
135}
136
137/*
138 * page is on free list, or is being allocated, verify that counters are zeroes
139 * crash if they are not.
140 */
141void __page_table_check_zero(struct page *page, unsigned int order)
142{
143	struct page_ext *page_ext;
144	unsigned long i;
145
146	BUG_ON(PageSlab(page));
147
148	page_ext = page_ext_get(page);
149
150	if (!page_ext)
151		return;
152
153	for (i = 0; i < (1ul << order); i++) {
154		struct page_table_check *ptc = get_page_table_check(page_ext);
155
156		BUG_ON(atomic_read(&ptc->anon_map_count));
157		BUG_ON(atomic_read(&ptc->file_map_count));
158		page_ext = page_ext_next(page_ext);
159	}
160	page_ext_put(page_ext);
161}
162
163void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
164{
165	if (&init_mm == mm)
166		return;
167
168	if (pte_user_accessible_page(pte)) {
169		page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
170	}
171}
172EXPORT_SYMBOL(__page_table_check_pte_clear);
173
174void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
175{
176	if (&init_mm == mm)
177		return;
178
179	if (pmd_user_accessible_page(pmd)) {
180		page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
181	}
182}
183EXPORT_SYMBOL(__page_table_check_pmd_clear);
184
185void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
186{
187	if (&init_mm == mm)
188		return;
189
190	if (pud_user_accessible_page(pud)) {
191		page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
192	}
193}
194EXPORT_SYMBOL(__page_table_check_pud_clear);
195
196/* Whether the swap entry cached writable information */
197static inline bool swap_cached_writable(swp_entry_t entry)
198{
199	return is_writable_device_exclusive_entry(entry) ||
200	    is_writable_device_private_entry(entry) ||
201	    is_writable_migration_entry(entry);
202}
203
204static inline void page_table_check_pte_flags(pte_t pte)
205{
206	if (pte_present(pte) && pte_uffd_wp(pte))
207		WARN_ON_ONCE(pte_write(pte));
208	else if (is_swap_pte(pte) && pte_swp_uffd_wp(pte))
209		WARN_ON_ONCE(swap_cached_writable(pte_to_swp_entry(pte)));
210}
211
212void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
213		unsigned int nr)
214{
215	unsigned int i;
216
217	if (&init_mm == mm)
218		return;
219
220	page_table_check_pte_flags(pte);
221
222	for (i = 0; i < nr; i++)
223		__page_table_check_pte_clear(mm, ptep_get(ptep + i));
224	if (pte_user_accessible_page(pte))
225		page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
226}
227EXPORT_SYMBOL(__page_table_check_ptes_set);
228
229static inline void page_table_check_pmd_flags(pmd_t pmd)
230{
231	if (pmd_present(pmd) && pmd_uffd_wp(pmd))
232		WARN_ON_ONCE(pmd_write(pmd));
233	else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd))
234		WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd)));
235}
236
237void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd)
238{
239	if (&init_mm == mm)
240		return;
241
242	page_table_check_pmd_flags(pmd);
243
244	__page_table_check_pmd_clear(mm, *pmdp);
245	if (pmd_user_accessible_page(pmd)) {
246		page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT,
247				     pmd_write(pmd));
248	}
249}
250EXPORT_SYMBOL(__page_table_check_pmd_set);
251
252void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud)
253{
254	if (&init_mm == mm)
255		return;
256
257	__page_table_check_pud_clear(mm, *pudp);
258	if (pud_user_accessible_page(pud)) {
259		page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT,
260				     pud_write(pud));
261	}
262}
263EXPORT_SYMBOL(__page_table_check_pud_set);
264
265void __page_table_check_pte_clear_range(struct mm_struct *mm,
266					unsigned long addr,
267					pmd_t pmd)
268{
269	if (&init_mm == mm)
270		return;
271
272	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
273		pte_t *ptep = pte_offset_map(&pmd, addr);
274		unsigned long i;
275
276		if (WARN_ON(!ptep))
277			return;
278		for (i = 0; i < PTRS_PER_PTE; i++) {
279			__page_table_check_pte_clear(mm, ptep_get(ptep));
280			addr += PAGE_SIZE;
281			ptep++;
282		}
283		pte_unmap(ptep - PTRS_PER_PTE);
284	}
285}