Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 1995-2005 Russell King
3// Copyright (C) 2012 ARM Ltd.
4// Copyright (C) 2013-2017 Andes Technology Corporation
5
6#include <linux/kernel.h>
7#include <linux/errno.h>
8#include <linux/swap.h>
9#include <linux/init.h>
10#include <linux/memblock.h>
11#include <linux/mman.h>
12#include <linux/nodemask.h>
13#include <linux/initrd.h>
14#include <linux/highmem.h>
15
16#include <asm/sections.h>
17#include <asm/setup.h>
18#include <asm/tlb.h>
19#include <asm/page.h>
20
21DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
22DEFINE_SPINLOCK(anon_alias_lock);
23extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
24
25/*
26 * empty_zero_page is a special page that is used for
27 * zero-initialized data and COW.
28 */
29struct page *empty_zero_page;
30EXPORT_SYMBOL(empty_zero_page);
31
32static void __init zone_sizes_init(void)
33{
34 unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
35
36 max_zone_pfn[ZONE_NORMAL] = max_low_pfn;
37#ifdef CONFIG_HIGHMEM
38 max_zone_pfn[ZONE_HIGHMEM] = max_pfn;
39#endif
40 free_area_init(max_zone_pfn);
41
42}
43
44/*
45 * Map all physical memory under high_memory into kernel's address space.
46 *
47 * This is explicitly coded for two-level page tables, so if you need
48 * something else then this needs to change.
49 */
50static void __init map_ram(void)
51{
52 unsigned long v, p, e;
53 pgd_t *pge;
54 p4d_t *p4e;
55 pud_t *pue;
56 pmd_t *pme;
57 pte_t *pte;
58 /* These mark extents of read-only kernel pages...
59 * ...from vmlinux.lds.S
60 */
61
62 p = (u32) memblock_start_of_DRAM() & PAGE_MASK;
63 e = min((u32) memblock_end_of_DRAM(), (u32) __pa(high_memory));
64
65 v = (u32) __va(p);
66 pge = pgd_offset_k(v);
67
68 while (p < e) {
69 int j;
70 p4e = p4d_offset(pge, v);
71 pue = pud_offset(p4e, v);
72 pme = pmd_offset(pue, v);
73
74 if ((u32) pue != (u32) pge || (u32) pme != (u32) pge) {
75 panic("%s: Kernel hardcoded for "
76 "two-level page tables", __func__);
77 }
78
79 /* Alloc one page for holding PTE's... */
80 pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
81 if (!pte)
82 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
83 __func__, PAGE_SIZE, PAGE_SIZE);
84 set_pmd(pme, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
85
86 /* Fill the newly allocated page with PTE'S */
87 for (j = 0; p < e && j < PTRS_PER_PTE;
88 v += PAGE_SIZE, p += PAGE_SIZE, j++, pte++) {
89 /* Create mapping between p and v. */
90 /* TODO: more fine grant for page access permission */
91 set_pte(pte, __pte(p + pgprot_val(PAGE_KERNEL)));
92 }
93
94 pge++;
95 }
96}
97static pmd_t *fixmap_pmd_p;
98static void __init fixedrange_init(void)
99{
100 unsigned long vaddr;
101 pmd_t *pmd;
102#ifdef CONFIG_HIGHMEM
103 pte_t *pte;
104#endif /* CONFIG_HIGHMEM */
105
106 /*
107 * Fixed mappings:
108 */
109 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1);
110 pmd = pmd_off_k(vaddr);
111 fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
112 if (!fixmap_pmd_p)
113 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
114 __func__, PAGE_SIZE, PAGE_SIZE);
115 set_pmd(pmd, __pmd(__pa(fixmap_pmd_p) + _PAGE_KERNEL_TABLE));
116
117#ifdef CONFIG_HIGHMEM
118 /*
119 * Permanent kmaps:
120 */
121 vaddr = PKMAP_BASE;
122
123 pmd = pmd_off_k(vaddr);
124 pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
125 if (!pte)
126 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
127 __func__, PAGE_SIZE, PAGE_SIZE);
128 set_pmd(pmd, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
129 pkmap_page_table = pte;
130#endif /* CONFIG_HIGHMEM */
131}
132
133/*
134 * paging_init() sets up the page tables, initialises the zone memory
135 * maps, and sets up the zero page, bad page and bad page tables.
136 */
137void __init paging_init(void)
138{
139 int i;
140 void *zero_page;
141
142 pr_info("Setting up paging and PTEs.\n");
143 /* clear out the init_mm.pgd that will contain the kernel's mappings */
144 for (i = 0; i < PTRS_PER_PGD; i++)
145 swapper_pg_dir[i] = __pgd(1);
146
147 map_ram();
148
149 fixedrange_init();
150
151 /* allocate space for empty_zero_page */
152 zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
153 if (!zero_page)
154 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
155 __func__, PAGE_SIZE, PAGE_SIZE);
156 zone_sizes_init();
157
158 empty_zero_page = virt_to_page(zero_page);
159 flush_dcache_page(empty_zero_page);
160}
161
162static inline void __init free_highmem(void)
163{
164#ifdef CONFIG_HIGHMEM
165 unsigned long pfn;
166 for (pfn = PFN_UP(__pa(high_memory)); pfn < max_pfn; pfn++) {
167 phys_addr_t paddr = (phys_addr_t) pfn << PAGE_SHIFT;
168 if (!memblock_is_reserved(paddr))
169 free_highmem_page(pfn_to_page(pfn));
170 }
171#endif
172}
173
174static void __init set_max_mapnr_init(void)
175{
176 max_mapnr = max_pfn;
177}
178
179/*
180 * mem_init() marks the free areas in the mem_map and tells us how much
181 * memory is free. This is done after various parts of the system have
182 * claimed their memory after the kernel image.
183 */
184void __init mem_init(void)
185{
186 phys_addr_t memory_start = memblock_start_of_DRAM();
187 BUG_ON(!mem_map);
188 set_max_mapnr_init();
189
190 free_highmem();
191
192 /* this will put all low memory onto the freelists */
193 memblock_free_all();
194 mem_init_print_info(NULL);
195
196 pr_info("virtual kernel memory layout:\n"
197 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
198#ifdef CONFIG_HIGHMEM
199 " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
200#endif
201 " consist : 0x%08lx - 0x%08lx (%4ld MB)\n"
202 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
203 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
204 " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
205 " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
206 " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
207 FIXADDR_START, FIXADDR_TOP, (FIXADDR_TOP - FIXADDR_START) >> 10,
208#ifdef CONFIG_HIGHMEM
209 PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE,
210 (LAST_PKMAP * PAGE_SIZE) >> 10,
211#endif
212 CONSISTENT_BASE, CONSISTENT_END,
213 ((CONSISTENT_END) - (CONSISTENT_BASE)) >> 20, VMALLOC_START,
214 (unsigned long)VMALLOC_END, (VMALLOC_END - VMALLOC_START) >> 20,
215 (unsigned long)__va(memory_start), (unsigned long)high_memory,
216 ((unsigned long)high_memory -
217 (unsigned long)__va(memory_start)) >> 20,
218 (unsigned long)&__init_begin, (unsigned long)&__init_end,
219 ((unsigned long)&__init_end -
220 (unsigned long)&__init_begin) >> 10, (unsigned long)&_etext,
221 (unsigned long)&_edata,
222 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
223 (unsigned long)&_text, (unsigned long)&_etext,
224 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
225
226 /*
227 * Check boundaries twice: Some fundamental inconsistencies can
228 * be detected at build time already.
229 */
230#ifdef CONFIG_HIGHMEM
231 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
232 BUILD_BUG_ON((CONSISTENT_END) > PKMAP_BASE);
233#endif
234 BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE);
235 BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END);
236
237#ifdef CONFIG_HIGHMEM
238 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
239 BUG_ON(CONSISTENT_END > PKMAP_BASE);
240#endif
241 BUG_ON(VMALLOC_END > CONSISTENT_BASE);
242 BUG_ON(VMALLOC_START >= VMALLOC_END);
243 BUG_ON((unsigned long)high_memory > VMALLOC_START);
244
245 return;
246}
247
248void __set_fixmap(enum fixed_addresses idx,
249 phys_addr_t phys, pgprot_t flags)
250{
251 unsigned long addr = __fix_to_virt(idx);
252 pte_t *pte;
253
254 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
255
256 pte = (pte_t *)&fixmap_pmd_p[pte_index(addr)];
257
258 if (pgprot_val(flags)) {
259 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
260 } else {
261 pte_clear(&init_mm, addr, pte);
262 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
263 }
264}