1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * This file contains kasan initialization code for ARM.
  4 *
  5 * Copyright (c) 2018 Samsung Electronics Co., Ltd.
  6 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
  7 * Author: Linus Walleij <linus.walleij@linaro.org>
  8 */
  9
 10#define pr_fmt(fmt) "kasan: " fmt
 11#include <linux/kasan.h>
 12#include <linux/kernel.h>
 13#include <linux/memblock.h>
 14#include <linux/sched/task.h>
 15#include <linux/start_kernel.h>
 16#include <linux/pgtable.h>
 17#include <asm/cputype.h>
 18#include <asm/highmem.h>
 19#include <asm/mach/map.h>
 20#include <asm/memory.h>
 21#include <asm/page.h>
 22#include <asm/pgalloc.h>
 23#include <asm/procinfo.h>
 24#include <asm/proc-fns.h>
 25
 26#include "mm.h"
 27
 28static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
 29
 30pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
 31
 32static __init void *kasan_alloc_block(size_t size)
 33{
 34	return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
 35				      MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
 36}
 37
 38static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
 39				      unsigned long end, bool early)
 40{
 41	unsigned long next;
 42	pte_t *ptep = pte_offset_kernel(pmdp, addr);
 43
 44	do {
 45		pte_t entry;
 46		void *p;
 47
 48		next = addr + PAGE_SIZE;
 49
 50		if (!early) {
 51			if (!pte_none(READ_ONCE(*ptep)))
 52				continue;
 53
 54			p = kasan_alloc_block(PAGE_SIZE);
 55			if (!p) {
 56				panic("%s failed to allocate shadow page for address 0x%lx\n",
 57				      __func__, addr);
 58				return;
 59			}
 60			memset(p, KASAN_SHADOW_INIT, PAGE_SIZE);
 61			entry = pfn_pte(virt_to_pfn(p),
 62					__pgprot(pgprot_val(PAGE_KERNEL)));
 63		} else if (pte_none(READ_ONCE(*ptep))) {
 64			/*
 65			 * The early shadow memory is mapping all KASan
 66			 * operations to one and the same page in memory,
 67			 * "kasan_early_shadow_page" so that the instrumentation
 68			 * will work on a scratch area until we can set up the
 69			 * proper KASan shadow memory.
 70			 */
 71			entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page),
 72					__pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN));
 73		} else {
 74			/*
 75			 * Early shadow mappings are PMD_SIZE aligned, so if the
 76			 * first entry is already set, they must all be set.
 77			 */
 78			return;
 79		}
 80
 81		set_pte_at(&init_mm, addr, ptep, entry);
 82	} while (ptep++, addr = next, addr != end);
 83}
 84
 85/*
 86 * The pmd (page middle directory) is only used on LPAE
 87 */
 88static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
 89				      unsigned long end, bool early)
 90{
 91	unsigned long next;
 92	pmd_t *pmdp = pmd_offset(pudp, addr);
 93
 94	do {
 95		if (pmd_none(*pmdp)) {
 96			/*
 97			 * We attempt to allocate a shadow block for the PMDs
 98			 * used by the PTEs for this address if it isn't already
 99			 * allocated.
100			 */
101			void *p = early ? kasan_early_shadow_pte :
102				kasan_alloc_block(PAGE_SIZE);
103
104			if (!p) {
105				panic("%s failed to allocate shadow block for address 0x%lx\n",
106				      __func__, addr);
107				return;
108			}
109			pmd_populate_kernel(&init_mm, pmdp, p);
110			flush_pmd_entry(pmdp);
111		}
112
113		next = pmd_addr_end(addr, end);
114		kasan_pte_populate(pmdp, addr, next, early);
115	} while (pmdp++, addr = next, addr != end);
116}
117
118static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
119				      bool early)
120{
121	unsigned long next;
122	pgd_t *pgdp;
123	p4d_t *p4dp;
124	pud_t *pudp;
125
126	pgdp = pgd_offset_k(addr);
127
128	do {
129		/*
130		 * Allocate and populate the shadow block of p4d folded into
131		 * pud folded into pmd if it doesn't already exist
132		 */
133		if (!early && pgd_none(*pgdp)) {
134			void *p = kasan_alloc_block(PAGE_SIZE);
135
136			if (!p) {
137				panic("%s failed to allocate shadow block for address 0x%lx\n",
138				      __func__, addr);
139				return;
140			}
141			pgd_populate(&init_mm, pgdp, p);
142		}
143
144		next = pgd_addr_end(addr, end);
145		/*
146		 * We just immediately jump over the p4d and pud page
147		 * directories since we believe ARM32 will never gain four
148		 * nor five level page tables.
149		 */
150		p4dp = p4d_offset(pgdp, addr);
151		pudp = pud_offset(p4dp, addr);
152
153		kasan_pmd_populate(pudp, addr, next, early);
154	} while (pgdp++, addr = next, addr != end);
155}
156
157extern struct proc_info_list *lookup_processor_type(unsigned int);
158
159void __init kasan_early_init(void)
160{
161	struct proc_info_list *list;
162
163	/*
164	 * locate processor in the list of supported processor
165	 * types.  The linker builds this table for us from the
166	 * entries in arch/arm/mm/proc-*.S
167	 */
168	list = lookup_processor_type(read_cpuid_id());
169	if (list) {
170#ifdef MULTI_CPU
171		processor = *list->proc;
172#endif
173	}
174
175	BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
176	/*
177	 * We walk the page table and set all of the shadow memory to point
178	 * to the scratch page.
179	 */
180	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true);
181}
182
183static void __init clear_pgds(unsigned long start,
184			unsigned long end)
185{
186	for (; start && start < end; start += PMD_SIZE)
187		pmd_clear(pmd_off_k(start));
188}
189
190static int __init create_mapping(void *start, void *end)
191{
192	void *shadow_start, *shadow_end;
193
194	shadow_start = kasan_mem_to_shadow(start);
195	shadow_end = kasan_mem_to_shadow(end);
196
197	pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n",
198		start, end, shadow_start, shadow_end);
199
200	kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK,
201			   PAGE_ALIGN((unsigned long)shadow_end), false);
202	return 0;
203}
204
205void __init kasan_init(void)
206{
207	phys_addr_t pa_start, pa_end;
208	u64 i;
209
210	/*
211	 * We are going to perform proper setup of shadow memory.
212	 *
213	 * At first we should unmap early shadow (clear_pgds() call bellow).
214	 * However, instrumented code can't execute without shadow memory.
215	 *
216	 * To keep the early shadow memory MMU tables around while setting up
217	 * the proper shadow memory, we copy swapper_pg_dir (the initial page
218	 * table) to tmp_pgd_table and use that to keep the early shadow memory
219	 * mapped until the full shadow setup is finished. Then we swap back
220	 * to the proper swapper_pg_dir.
221	 */
222
223	memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
224#ifdef CONFIG_ARM_LPAE
225	/* We need to be in the same PGD or this won't work */
226	BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
227		     pgd_index(KASAN_SHADOW_END));
228	memcpy(tmp_pmd_table,
229	       (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
230	       sizeof(tmp_pmd_table));
231	set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
232		__pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
233#endif
234	cpu_switch_mm(tmp_pgd_table, &init_mm);
235	local_flush_tlb_all();
236
237	clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
238
239	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
240		kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
241					    kasan_mem_to_shadow((void *)VMALLOC_END));
242
243	kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_END),
244				    kasan_mem_to_shadow((void *)-1UL) + 1);
245
246	for_each_mem_range(i, &pa_start, &pa_end) {
247		void *start = __va(pa_start);
248		void *end = __va(pa_end);
249
250		/* Do not attempt to shadow highmem */
251		if (pa_start >= arm_lowmem_limit) {
252			pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end);
253			continue;
254		}
255		if (pa_end > arm_lowmem_limit) {
256			pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n",
257				&pa_start, &pa_end, &arm_lowmem_limit);
258			end = __va(arm_lowmem_limit);
259		}
260		if (start >= end) {
261			pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
262				&pa_start, &pa_end, start, end);
263			continue;
264		}
265
266		create_mapping(start, end);
267	}
268
269	/*
270	 * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
271	 *    so we need to map this area if CONFIG_KASAN_VMALLOC=n. With
272	 *    VMALLOC support KASAN will manage this region dynamically,
273	 *    refer to kasan_populate_vmalloc() and ARM's implementation of
274	 *    module_alloc().
275	 * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
276	 *    ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
277	 *    use kasan_populate_zero_shadow.
278	 */
279	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES))
280		create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END));
281	create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE));
282
283	/*
284	 * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
285	 * we should make sure that it maps the zero page read-only.
286	 */
287	for (i = 0; i < PTRS_PER_PTE; i++)
288		set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
289			   &kasan_early_shadow_pte[i],
290			   pfn_pte(virt_to_pfn(kasan_early_shadow_page),
291				__pgprot(pgprot_val(PAGE_KERNEL)
292					 | L_PTE_RDONLY)));
293
294	cpu_switch_mm(swapper_pg_dir, &init_mm);
295	local_flush_tlb_all();
296
297	memset(kasan_early_shadow_page, 0, PAGE_SIZE);
298	pr_info("Kernel address sanitizer initialized\n");
299	init_task.kasan_depth = 0;
300}