1/*
  2 * This file contains some kasan initialization code.
  3 *
  4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
  5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
  6 *
  7 * This program is free software; you can redistribute it and/or modify
  8 * it under the terms of the GNU General Public License version 2 as
  9 * published by the Free Software Foundation.
 10 *
 11 */
 12
 13#include <linux/bootmem.h>
 14#include <linux/init.h>
 15#include <linux/kasan.h>
 16#include <linux/kernel.h>
 17#include <linux/memblock.h>
 18#include <linux/pfn.h>
 19
 20#include <asm/page.h>
 21#include <asm/pgalloc.h>
 22
 23/*
 24 * This page serves two purposes:
 25 *   - It used as early shadow memory. The entire shadow region populated
 26 *     with this page, before we will be able to setup normal shadow memory.
 27 *   - Latter it reused it as zero shadow to cover large ranges of memory
 28 *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
 29 */
 30unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
 31
 32#if CONFIG_PGTABLE_LEVELS > 3
 33pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
 34#endif
 35#if CONFIG_PGTABLE_LEVELS > 2
 36pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
 37#endif
 38pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
 39
 40static __init void *early_alloc(size_t size, int node)
 41{
 42	return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
 43					BOOTMEM_ALLOC_ACCESSIBLE, node);
 44}
 45
 46static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
 47				unsigned long end)
 48{
 49	pte_t *pte = pte_offset_kernel(pmd, addr);
 50	pte_t zero_pte;
 51
 52	zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL);
 53	zero_pte = pte_wrprotect(zero_pte);
 54
 55	while (addr + PAGE_SIZE <= end) {
 56		set_pte_at(&init_mm, addr, pte, zero_pte);
 57		addr += PAGE_SIZE;
 58		pte = pte_offset_kernel(pmd, addr);
 59	}
 60}
 61
 62static void __init zero_pmd_populate(pud_t *pud, unsigned long addr,
 63				unsigned long end)
 64{
 65	pmd_t *pmd = pmd_offset(pud, addr);
 66	unsigned long next;
 67
 68	do {
 69		next = pmd_addr_end(addr, end);
 70
 71		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
 72			pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
 73			continue;
 74		}
 75
 76		if (pmd_none(*pmd)) {
 77			pmd_populate_kernel(&init_mm, pmd,
 78					early_alloc(PAGE_SIZE, NUMA_NO_NODE));
 79		}
 80		zero_pte_populate(pmd, addr, next);
 81	} while (pmd++, addr = next, addr != end);
 82}
 83
 84static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
 85				unsigned long end)
 86{
 87	pud_t *pud = pud_offset(pgd, addr);
 88	unsigned long next;
 89
 90	do {
 91		next = pud_addr_end(addr, end);
 92		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
 93			pmd_t *pmd;
 94
 95			pud_populate(&init_mm, pud, kasan_zero_pmd);
 96			pmd = pmd_offset(pud, addr);
 97			pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
 98			continue;
 99		}
100
101		if (pud_none(*pud)) {
102			pud_populate(&init_mm, pud,
103				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
104		}
105		zero_pmd_populate(pud, addr, next);
106	} while (pud++, addr = next, addr != end);
107}
108
109/**
110 * kasan_populate_zero_shadow - populate shadow memory region with
111 *                               kasan_zero_page
112 * @shadow_start - start of the memory range to populate
113 * @shadow_end   - end of the memory range to populate
114 */
115void __init kasan_populate_zero_shadow(const void *shadow_start,
116				const void *shadow_end)
117{
118	unsigned long addr = (unsigned long)shadow_start;
119	unsigned long end = (unsigned long)shadow_end;
120	pgd_t *pgd = pgd_offset_k(addr);
121	unsigned long next;
122
123	do {
124		next = pgd_addr_end(addr, end);
125
126		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
127			pud_t *pud;
128			pmd_t *pmd;
129
130			/*
131			 * kasan_zero_pud should be populated with pmds
132			 * at this moment.
133			 * [pud,pmd]_populate*() below needed only for
134			 * 3,2 - level page tables where we don't have
135			 * puds,pmds, so pgd_populate(), pud_populate()
136			 * is noops.
137			 */
138			pgd_populate(&init_mm, pgd, kasan_zero_pud);
139			pud = pud_offset(pgd, addr);
140			pud_populate(&init_mm, pud, kasan_zero_pmd);
141			pmd = pmd_offset(pud, addr);
142			pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
143			continue;
144		}
145
146		if (pgd_none(*pgd)) {
147			pgd_populate(&init_mm, pgd,
148				early_alloc(PAGE_SIZE, NUMA_NO_NODE));
149		}
150		zero_pud_populate(pgd, addr, next);
151	} while (pgd++, addr = next, addr != end);
152}