1/*
  2 * This file contains some kasan initialization code.
  3 *
  4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
  5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
  6 *
  7 * This program is free software; you can redistribute it and/or modify
  8 * it under the terms of the GNU General Public License version 2 as
  9 * published by the Free Software Foundation.
 10 *
 11 */
 12
 13#include <linux/bootmem.h>
 14#include <linux/init.h>
 15#include <linux/kasan.h>
 16#include <linux/kernel.h>
 17#include <linux/memblock.h>
 18#include <linux/mm.h>
 19#include <linux/pfn.h>
 20
 21#include <asm/page.h>
 22#include <asm/pgalloc.h>
 23
 24/*
 25 * This page serves two purposes:
 26 *   - It used as early shadow memory. The entire shadow region populated
 27 *     with this page, before we will be able to setup normal shadow memory.
 28 *   - Latter it reused it as zero shadow to cover large ranges of memory
 29 *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
 30 */
 31unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
 32
 33#if CONFIG_PGTABLE_LEVELS > 4
 34p4d_t kasan_zero_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss;
 35#endif
 36#if CONFIG_PGTABLE_LEVELS > 3
 37pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
 38#endif
 39#if CONFIG_PGTABLE_LEVELS > 2
 40pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
 41#endif
 42pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
 43
 44static __init void *early_alloc(size_t size, int node)
 45{
 46	return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
 47					BOOTMEM_ALLOC_ACCESSIBLE, node);
 48}
 49
 50static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
 51				unsigned long end)
 52{
 53	pte_t *pte = pte_offset_kernel(pmd, addr);
 54	pte_t zero_pte;
 55
 56	zero_pte = pfn_pte(PFN_DOWN(__pa_symbol(kasan_zero_page)), PAGE_KERNEL);
 57	zero_pte = pte_wrprotect(zero_pte);
 58
 59	while (addr + PAGE_SIZE <= end) {
 60		set_pte_at(&init_mm, addr, pte, zero_pte);
 61		addr += PAGE_SIZE;
 62		pte = pte_offset_kernel(pmd, addr);
 63	}
 64}
 65
 66static void __init zero_pmd_populate(pud_t *pud, unsigned long addr,
 67				unsigned long end)
 68{
 69	pmd_t *pmd = pmd_offset(pud, addr);
 70	unsigned long next;
 71
 72	do {
 73		next = pmd_addr_end(addr, end);
 74
 75		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
 76			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
 77			continue;
 78		}
 79
 80		if (pmd_none(*pmd)) {
 81			pmd_populate_kernel(&init_mm, pmd,
 82					early_alloc(PAGE_SIZE, NUMA_NO_NODE));
 83		}
 84		zero_pte_populate(pmd, addr, next);
 85	} while (pmd++, addr = next, addr != end);
 86}
 87
 88static void __init zero_pud_populate(p4d_t *p4d, unsigned long addr,
 89				unsigned long end)
 90{
 91	pud_t *pud = pud_offset(p4d, addr);
 92	unsigned long next;
 93
 94	do {
 95		next = pud_addr_end(addr, end);
 96		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
 97			pmd_t *pmd;
 98
 99			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
100			pmd = pmd_offset(pud, addr);
101			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
102			continue;
103		}
104
105		if (pud_none(*pud)) {
106			pud_populate(&init_mm, pud,
107				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
108		}
109		zero_pmd_populate(pud, addr, next);
110	} while (pud++, addr = next, addr != end);
111}
112
113static void __init zero_p4d_populate(pgd_t *pgd, unsigned long addr,
114				unsigned long end)
115{
116	p4d_t *p4d = p4d_offset(pgd, addr);
117	unsigned long next;
118
119	do {
120		next = p4d_addr_end(addr, end);
121		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {
122			pud_t *pud;
123			pmd_t *pmd;
124
125			p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
126			pud = pud_offset(p4d, addr);
127			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
128			pmd = pmd_offset(pud, addr);
129			pmd_populate_kernel(&init_mm, pmd,
130						lm_alias(kasan_zero_pte));
131			continue;
132		}
133
134		if (p4d_none(*p4d)) {
135			p4d_populate(&init_mm, p4d,
136				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
137		}
138		zero_pud_populate(p4d, addr, next);
139	} while (p4d++, addr = next, addr != end);
140}
141
142/**
143 * kasan_populate_zero_shadow - populate shadow memory region with
144 *                               kasan_zero_page
145 * @shadow_start - start of the memory range to populate
146 * @shadow_end   - end of the memory range to populate
147 */
148void __init kasan_populate_zero_shadow(const void *shadow_start,
149				const void *shadow_end)
150{
151	unsigned long addr = (unsigned long)shadow_start;
152	unsigned long end = (unsigned long)shadow_end;
153	pgd_t *pgd = pgd_offset_k(addr);
154	unsigned long next;
155
156	do {
157		next = pgd_addr_end(addr, end);
158
159		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
160			p4d_t *p4d;
161			pud_t *pud;
162			pmd_t *pmd;
163
164			/*
165			 * kasan_zero_pud should be populated with pmds
166			 * at this moment.
167			 * [pud,pmd]_populate*() below needed only for
168			 * 3,2 - level page tables where we don't have
169			 * puds,pmds, so pgd_populate(), pud_populate()
170			 * is noops.
171			 *
172			 * The ifndef is required to avoid build breakage.
173			 *
174			 * With 5level-fixup.h, pgd_populate() is not nop and
175			 * we reference kasan_zero_p4d. It's not defined
176			 * unless 5-level paging enabled.
177			 *
178			 * The ifndef can be dropped once all KASAN-enabled
179			 * architectures will switch to pgtable-nop4d.h.
180			 */
181#ifndef __ARCH_HAS_5LEVEL_HACK
182			pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d));
183#endif
184			p4d = p4d_offset(pgd, addr);
185			p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud));
186			pud = pud_offset(p4d, addr);
187			pud_populate(&init_mm, pud, lm_alias(kasan_zero_pmd));
188			pmd = pmd_offset(pud, addr);
189			pmd_populate_kernel(&init_mm, pmd, lm_alias(kasan_zero_pte));
190			continue;
191		}
192
193		if (pgd_none(*pgd)) {
194			pgd_populate(&init_mm, pgd,
195				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
196		}
197		zero_p4d_populate(pgd, addr, next);
198	} while (pgd++, addr = next, addr != end);
199}