Loading...
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 1995-2005 Russell King
3// Copyright (C) 2012 ARM Ltd.
4// Copyright (C) 2013-2017 Andes Technology Corporation
5
6#include <linux/kernel.h>
7#include <linux/errno.h>
8#include <linux/swap.h>
9#include <linux/init.h>
10#include <linux/memblock.h>
11#include <linux/mman.h>
12#include <linux/nodemask.h>
13#include <linux/initrd.h>
14#include <linux/highmem.h>
15
16#include <asm/sections.h>
17#include <asm/setup.h>
18#include <asm/tlb.h>
19#include <asm/page.h>
20
21DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
22DEFINE_SPINLOCK(anon_alias_lock);
23extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
24
25/*
26 * empty_zero_page is a special page that is used for
27 * zero-initialized data and COW.
28 */
29struct page *empty_zero_page;
30EXPORT_SYMBOL(empty_zero_page);
31
32static void __init zone_sizes_init(void)
33{
34 unsigned long zones_size[MAX_NR_ZONES];
35
36 /* Clear the zone sizes */
37 memset(zones_size, 0, sizeof(zones_size));
38
39 zones_size[ZONE_NORMAL] = max_low_pfn;
40#ifdef CONFIG_HIGHMEM
41 zones_size[ZONE_HIGHMEM] = max_pfn;
42#endif
43 free_area_init(zones_size);
44
45}
46
47/*
48 * Map all physical memory under high_memory into kernel's address space.
49 *
50 * This is explicitly coded for two-level page tables, so if you need
51 * something else then this needs to change.
52 */
53static void __init map_ram(void)
54{
55 unsigned long v, p, e;
56 pgd_t *pge;
57 pud_t *pue;
58 pmd_t *pme;
59 pte_t *pte;
60 /* These mark extents of read-only kernel pages...
61 * ...from vmlinux.lds.S
62 */
63
64 p = (u32) memblock_start_of_DRAM() & PAGE_MASK;
65 e = min((u32) memblock_end_of_DRAM(), (u32) __pa(high_memory));
66
67 v = (u32) __va(p);
68 pge = pgd_offset_k(v);
69
70 while (p < e) {
71 int j;
72 pue = pud_offset(pge, v);
73 pme = pmd_offset(pue, v);
74
75 if ((u32) pue != (u32) pge || (u32) pme != (u32) pge) {
76 panic("%s: Kernel hardcoded for "
77 "two-level page tables", __func__);
78 }
79
80 /* Alloc one page for holding PTE's... */
81 pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
82 if (!pte)
83 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
84 __func__, PAGE_SIZE, PAGE_SIZE);
85 set_pmd(pme, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
86
87 /* Fill the newly allocated page with PTE'S */
88 for (j = 0; p < e && j < PTRS_PER_PTE;
89 v += PAGE_SIZE, p += PAGE_SIZE, j++, pte++) {
90 /* Create mapping between p and v. */
91 /* TODO: more fine grant for page access permission */
92 set_pte(pte, __pte(p + pgprot_val(PAGE_KERNEL)));
93 }
94
95 pge++;
96 }
97}
98static pmd_t *fixmap_pmd_p;
99static void __init fixedrange_init(void)
100{
101 unsigned long vaddr;
102 pgd_t *pgd;
103 pud_t *pud;
104 pmd_t *pmd;
105#ifdef CONFIG_HIGHMEM
106 pte_t *pte;
107#endif /* CONFIG_HIGHMEM */
108
109 /*
110 * Fixed mappings:
111 */
112 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1);
113 pgd = swapper_pg_dir + pgd_index(vaddr);
114 pud = pud_offset(pgd, vaddr);
115 pmd = pmd_offset(pud, vaddr);
116 fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
117 if (!fixmap_pmd_p)
118 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
119 __func__, PAGE_SIZE, PAGE_SIZE);
120 set_pmd(pmd, __pmd(__pa(fixmap_pmd_p) + _PAGE_KERNEL_TABLE));
121
122#ifdef CONFIG_HIGHMEM
123 /*
124 * Permanent kmaps:
125 */
126 vaddr = PKMAP_BASE;
127
128 pgd = swapper_pg_dir + pgd_index(vaddr);
129 pud = pud_offset(pgd, vaddr);
130 pmd = pmd_offset(pud, vaddr);
131 pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
132 if (!pte)
133 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
134 __func__, PAGE_SIZE, PAGE_SIZE);
135 set_pmd(pmd, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
136 pkmap_page_table = pte;
137#endif /* CONFIG_HIGHMEM */
138}
139
140/*
141 * paging_init() sets up the page tables, initialises the zone memory
142 * maps, and sets up the zero page, bad page and bad page tables.
143 */
144void __init paging_init(void)
145{
146 int i;
147 void *zero_page;
148
149 pr_info("Setting up paging and PTEs.\n");
150 /* clear out the init_mm.pgd that will contain the kernel's mappings */
151 for (i = 0; i < PTRS_PER_PGD; i++)
152 swapper_pg_dir[i] = __pgd(1);
153
154 map_ram();
155
156 fixedrange_init();
157
158 /* allocate space for empty_zero_page */
159 zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
160 if (!zero_page)
161 panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
162 __func__, PAGE_SIZE, PAGE_SIZE);
163 zone_sizes_init();
164
165 empty_zero_page = virt_to_page(zero_page);
166 flush_dcache_page(empty_zero_page);
167}
168
169static inline void __init free_highmem(void)
170{
171#ifdef CONFIG_HIGHMEM
172 unsigned long pfn;
173 for (pfn = PFN_UP(__pa(high_memory)); pfn < max_pfn; pfn++) {
174 phys_addr_t paddr = (phys_addr_t) pfn << PAGE_SHIFT;
175 if (!memblock_is_reserved(paddr))
176 free_highmem_page(pfn_to_page(pfn));
177 }
178#endif
179}
180
181static void __init set_max_mapnr_init(void)
182{
183 max_mapnr = max_pfn;
184}
185
186/*
187 * mem_init() marks the free areas in the mem_map and tells us how much
188 * memory is free. This is done after various parts of the system have
189 * claimed their memory after the kernel image.
190 */
191void __init mem_init(void)
192{
193 phys_addr_t memory_start = memblock_start_of_DRAM();
194 BUG_ON(!mem_map);
195 set_max_mapnr_init();
196
197 free_highmem();
198
199 /* this will put all low memory onto the freelists */
200 memblock_free_all();
201 mem_init_print_info(NULL);
202
203 pr_info("virtual kernel memory layout:\n"
204 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
205#ifdef CONFIG_HIGHMEM
206 " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
207#endif
208 " consist : 0x%08lx - 0x%08lx (%4ld MB)\n"
209 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
210 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
211 " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
212 " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
213 " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
214 FIXADDR_START, FIXADDR_TOP, (FIXADDR_TOP - FIXADDR_START) >> 10,
215#ifdef CONFIG_HIGHMEM
216 PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE,
217 (LAST_PKMAP * PAGE_SIZE) >> 10,
218#endif
219 CONSISTENT_BASE, CONSISTENT_END,
220 ((CONSISTENT_END) - (CONSISTENT_BASE)) >> 20, VMALLOC_START,
221 (unsigned long)VMALLOC_END, (VMALLOC_END - VMALLOC_START) >> 20,
222 (unsigned long)__va(memory_start), (unsigned long)high_memory,
223 ((unsigned long)high_memory -
224 (unsigned long)__va(memory_start)) >> 20,
225 (unsigned long)&__init_begin, (unsigned long)&__init_end,
226 ((unsigned long)&__init_end -
227 (unsigned long)&__init_begin) >> 10, (unsigned long)&_etext,
228 (unsigned long)&_edata,
229 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
230 (unsigned long)&_text, (unsigned long)&_etext,
231 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
232
233 /*
234 * Check boundaries twice: Some fundamental inconsistencies can
235 * be detected at build time already.
236 */
237#ifdef CONFIG_HIGHMEM
238 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
239 BUILD_BUG_ON((CONSISTENT_END) > PKMAP_BASE);
240#endif
241 BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE);
242 BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END);
243
244#ifdef CONFIG_HIGHMEM
245 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
246 BUG_ON(CONSISTENT_END > PKMAP_BASE);
247#endif
248 BUG_ON(VMALLOC_END > CONSISTENT_BASE);
249 BUG_ON(VMALLOC_START >= VMALLOC_END);
250 BUG_ON((unsigned long)high_memory > VMALLOC_START);
251
252 return;
253}
254
255void __set_fixmap(enum fixed_addresses idx,
256 phys_addr_t phys, pgprot_t flags)
257{
258 unsigned long addr = __fix_to_virt(idx);
259 pte_t *pte;
260
261 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
262
263 pte = (pte_t *)&fixmap_pmd_p[pte_index(addr)];
264
265 if (pgprot_val(flags)) {
266 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
267 } else {
268 pte_clear(&init_mm, addr, pte);
269 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
270 }
271}
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 1995-2005 Russell King
3// Copyright (C) 2012 ARM Ltd.
4// Copyright (C) 2013-2017 Andes Technology Corporation
5
6#include <linux/kernel.h>
7#include <linux/errno.h>
8#include <linux/swap.h>
9#include <linux/init.h>
10#include <linux/bootmem.h>
11#include <linux/mman.h>
12#include <linux/nodemask.h>
13#include <linux/initrd.h>
14#include <linux/highmem.h>
15#include <linux/memblock.h>
16
17#include <asm/sections.h>
18#include <asm/setup.h>
19#include <asm/tlb.h>
20#include <asm/page.h>
21
22DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
23DEFINE_SPINLOCK(anon_alias_lock);
24extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
25extern unsigned long phys_initrd_start;
26extern unsigned long phys_initrd_size;
27
28/*
29 * empty_zero_page is a special page that is used for
30 * zero-initialized data and COW.
31 */
32struct page *empty_zero_page;
33EXPORT_SYMBOL(empty_zero_page);
34
35static void __init zone_sizes_init(void)
36{
37 unsigned long zones_size[MAX_NR_ZONES];
38
39 /* Clear the zone sizes */
40 memset(zones_size, 0, sizeof(zones_size));
41
42 zones_size[ZONE_NORMAL] = max_low_pfn;
43#ifdef CONFIG_HIGHMEM
44 zones_size[ZONE_HIGHMEM] = max_pfn;
45#endif
46 free_area_init(zones_size);
47
48}
49
50/*
51 * Map all physical memory under high_memory into kernel's address space.
52 *
53 * This is explicitly coded for two-level page tables, so if you need
54 * something else then this needs to change.
55 */
56static void __init map_ram(void)
57{
58 unsigned long v, p, e;
59 pgd_t *pge;
60 pud_t *pue;
61 pmd_t *pme;
62 pte_t *pte;
63 /* These mark extents of read-only kernel pages...
64 * ...from vmlinux.lds.S
65 */
66
67 p = (u32) memblock_start_of_DRAM() & PAGE_MASK;
68 e = min((u32) memblock_end_of_DRAM(), (u32) __pa(high_memory));
69
70 v = (u32) __va(p);
71 pge = pgd_offset_k(v);
72
73 while (p < e) {
74 int j;
75 pue = pud_offset(pge, v);
76 pme = pmd_offset(pue, v);
77
78 if ((u32) pue != (u32) pge || (u32) pme != (u32) pge) {
79 panic("%s: Kernel hardcoded for "
80 "two-level page tables", __func__);
81 }
82
83 /* Alloc one page for holding PTE's... */
84 pte = (pte_t *) __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
85 memset(pte, 0, PAGE_SIZE);
86 set_pmd(pme, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
87
88 /* Fill the newly allocated page with PTE'S */
89 for (j = 0; p < e && j < PTRS_PER_PTE;
90 v += PAGE_SIZE, p += PAGE_SIZE, j++, pte++) {
91 /* Create mapping between p and v. */
92 /* TODO: more fine grant for page access permission */
93 set_pte(pte, __pte(p + pgprot_val(PAGE_KERNEL)));
94 }
95
96 pge++;
97 }
98}
99static pmd_t *fixmap_pmd_p;
100static void __init fixedrange_init(void)
101{
102 unsigned long vaddr;
103 pgd_t *pgd;
104 pud_t *pud;
105 pmd_t *pmd;
106#ifdef CONFIG_HIGHMEM
107 pte_t *pte;
108#endif /* CONFIG_HIGHMEM */
109
110 /*
111 * Fixed mappings:
112 */
113 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1);
114 pgd = swapper_pg_dir + pgd_index(vaddr);
115 pud = pud_offset(pgd, vaddr);
116 pmd = pmd_offset(pud, vaddr);
117 fixmap_pmd_p = (pmd_t *) __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
118 memset(fixmap_pmd_p, 0, PAGE_SIZE);
119 set_pmd(pmd, __pmd(__pa(fixmap_pmd_p) + _PAGE_KERNEL_TABLE));
120
121#ifdef CONFIG_HIGHMEM
122 /*
123 * Permanent kmaps:
124 */
125 vaddr = PKMAP_BASE;
126
127 pgd = swapper_pg_dir + pgd_index(vaddr);
128 pud = pud_offset(pgd, vaddr);
129 pmd = pmd_offset(pud, vaddr);
130 pte = (pte_t *) __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
131 memset(pte, 0, PAGE_SIZE);
132 set_pmd(pmd, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE));
133 pkmap_page_table = pte;
134#endif /* CONFIG_HIGHMEM */
135}
136
137/*
138 * paging_init() sets up the page tables, initialises the zone memory
139 * maps, and sets up the zero page, bad page and bad page tables.
140 */
141void __init paging_init(void)
142{
143 int i;
144 void *zero_page;
145
146 pr_info("Setting up paging and PTEs.\n");
147 /* clear out the init_mm.pgd that will contain the kernel's mappings */
148 for (i = 0; i < PTRS_PER_PGD; i++)
149 swapper_pg_dir[i] = __pgd(1);
150
151 map_ram();
152
153 fixedrange_init();
154
155 /* allocate space for empty_zero_page */
156 zero_page = __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
157 memset(zero_page, 0, PAGE_SIZE);
158 zone_sizes_init();
159
160 empty_zero_page = virt_to_page(zero_page);
161 flush_dcache_page(empty_zero_page);
162}
163
164static inline void __init free_highmem(void)
165{
166#ifdef CONFIG_HIGHMEM
167 unsigned long pfn;
168 for (pfn = PFN_UP(__pa(high_memory)); pfn < max_pfn; pfn++) {
169 phys_addr_t paddr = (phys_addr_t) pfn << PAGE_SHIFT;
170 if (!memblock_is_reserved(paddr))
171 free_highmem_page(pfn_to_page(pfn));
172 }
173#endif
174}
175
176static void __init set_max_mapnr_init(void)
177{
178 max_mapnr = max_pfn;
179}
180
181/*
182 * mem_init() marks the free areas in the mem_map and tells us how much
183 * memory is free. This is done after various parts of the system have
184 * claimed their memory after the kernel image.
185 */
186void __init mem_init(void)
187{
188 phys_addr_t memory_start = memblock_start_of_DRAM();
189 BUG_ON(!mem_map);
190 set_max_mapnr_init();
191
192 free_highmem();
193
194 /* this will put all low memory onto the freelists */
195 free_all_bootmem();
196 mem_init_print_info(NULL);
197
198 pr_info("virtual kernel memory layout:\n"
199 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
200#ifdef CONFIG_HIGHMEM
201 " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
202#endif
203 " consist : 0x%08lx - 0x%08lx (%4ld MB)\n"
204 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
205 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
206 " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
207 " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
208 " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
209 FIXADDR_START, FIXADDR_TOP, (FIXADDR_TOP - FIXADDR_START) >> 10,
210#ifdef CONFIG_HIGHMEM
211 PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE,
212 (LAST_PKMAP * PAGE_SIZE) >> 10,
213#endif
214 CONSISTENT_BASE, CONSISTENT_END,
215 ((CONSISTENT_END) - (CONSISTENT_BASE)) >> 20, VMALLOC_START,
216 (unsigned long)VMALLOC_END, (VMALLOC_END - VMALLOC_START) >> 20,
217 (unsigned long)__va(memory_start), (unsigned long)high_memory,
218 ((unsigned long)high_memory -
219 (unsigned long)__va(memory_start)) >> 20,
220 (unsigned long)&__init_begin, (unsigned long)&__init_end,
221 ((unsigned long)&__init_end -
222 (unsigned long)&__init_begin) >> 10, (unsigned long)&_etext,
223 (unsigned long)&_edata,
224 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
225 (unsigned long)&_text, (unsigned long)&_etext,
226 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
227
228 /*
229 * Check boundaries twice: Some fundamental inconsistencies can
230 * be detected at build time already.
231 */
232#ifdef CONFIG_HIGHMEM
233 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
234 BUILD_BUG_ON((CONSISTENT_END) > PKMAP_BASE);
235#endif
236 BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE);
237 BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END);
238
239#ifdef CONFIG_HIGHMEM
240 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > FIXADDR_START);
241 BUG_ON(CONSISTENT_END > PKMAP_BASE);
242#endif
243 BUG_ON(VMALLOC_END > CONSISTENT_BASE);
244 BUG_ON(VMALLOC_START >= VMALLOC_END);
245 BUG_ON((unsigned long)high_memory > VMALLOC_START);
246
247 return;
248}
249
250void free_initmem(void)
251{
252 free_initmem_default(-1);
253}
254
255#ifdef CONFIG_BLK_DEV_INITRD
256void free_initrd_mem(unsigned long start, unsigned long end)
257{
258 free_reserved_area((void *)start, (void *)end, -1, "initrd");
259}
260#endif
261
262void __set_fixmap(enum fixed_addresses idx,
263 phys_addr_t phys, pgprot_t flags)
264{
265 unsigned long addr = __fix_to_virt(idx);
266 pte_t *pte;
267
268 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
269
270 pte = (pte_t *)&fixmap_pmd_p[pte_index(addr)];;
271
272 if (pgprot_val(flags)) {
273 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
274 } else {
275 pte_clear(&init_mm, addr, pte);
276 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
277 }
278}