Loading...
1/*
2 * Based on arch/arm/mm/init.c
3 *
4 * Copyright (C) 1995-2005 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/kernel.h>
21#include <linux/export.h>
22#include <linux/errno.h>
23#include <linux/swap.h>
24#include <linux/init.h>
25#include <linux/bootmem.h>
26#include <linux/mman.h>
27#include <linux/nodemask.h>
28#include <linux/initrd.h>
29#include <linux/gfp.h>
30#include <linux/memblock.h>
31#include <linux/sort.h>
32#include <linux/of_fdt.h>
33#include <linux/dma-mapping.h>
34#include <linux/dma-contiguous.h>
35#include <linux/efi.h>
36#include <linux/swiotlb.h>
37
38#include <asm/boot.h>
39#include <asm/fixmap.h>
40#include <asm/kasan.h>
41#include <asm/kernel-pgtable.h>
42#include <asm/memory.h>
43#include <asm/sections.h>
44#include <asm/setup.h>
45#include <asm/sizes.h>
46#include <asm/tlb.h>
47#include <asm/alternative.h>
48
49#include "mm.h"
50
51/*
52 * We need to be able to catch inadvertent references to memstart_addr
53 * that occur (potentially in generic code) before arm64_memblock_init()
54 * executes, which assigns it its actual value. So use a default value
55 * that cannot be mistaken for a real physical address.
56 */
57s64 memstart_addr __read_mostly = -1;
58phys_addr_t arm64_dma_phys_limit __read_mostly;
59
60#ifdef CONFIG_BLK_DEV_INITRD
61static int __init early_initrd(char *p)
62{
63 unsigned long start, size;
64 char *endp;
65
66 start = memparse(p, &endp);
67 if (*endp == ',') {
68 size = memparse(endp + 1, NULL);
69
70 initrd_start = start;
71 initrd_end = start + size;
72 }
73 return 0;
74}
75early_param("initrd", early_initrd);
76#endif
77
78/*
79 * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
80 * currently assumes that for memory starting above 4G, 32-bit devices will
81 * use a DMA offset.
82 */
83static phys_addr_t __init max_zone_dma_phys(void)
84{
85 phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32);
86 return min(offset + (1ULL << 32), memblock_end_of_DRAM());
87}
88
89static void __init zone_sizes_init(unsigned long min, unsigned long max)
90{
91 struct memblock_region *reg;
92 unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
93 unsigned long max_dma = min;
94
95 memset(zone_size, 0, sizeof(zone_size));
96
97 /* 4GB maximum for 32-bit only capable devices */
98#ifdef CONFIG_ZONE_DMA
99 max_dma = PFN_DOWN(arm64_dma_phys_limit);
100 zone_size[ZONE_DMA] = max_dma - min;
101#endif
102 zone_size[ZONE_NORMAL] = max - max_dma;
103
104 memcpy(zhole_size, zone_size, sizeof(zhole_size));
105
106 for_each_memblock(memory, reg) {
107 unsigned long start = memblock_region_memory_base_pfn(reg);
108 unsigned long end = memblock_region_memory_end_pfn(reg);
109
110 if (start >= max)
111 continue;
112
113#ifdef CONFIG_ZONE_DMA
114 if (start < max_dma) {
115 unsigned long dma_end = min(end, max_dma);
116 zhole_size[ZONE_DMA] -= dma_end - start;
117 }
118#endif
119 if (end > max_dma) {
120 unsigned long normal_end = min(end, max);
121 unsigned long normal_start = max(start, max_dma);
122 zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
123 }
124 }
125
126 free_area_init_node(0, zone_size, min, zhole_size);
127}
128
129#ifdef CONFIG_HAVE_ARCH_PFN_VALID
130int pfn_valid(unsigned long pfn)
131{
132 return memblock_is_map_memory(pfn << PAGE_SHIFT);
133}
134EXPORT_SYMBOL(pfn_valid);
135#endif
136
137#ifndef CONFIG_SPARSEMEM
138static void __init arm64_memory_present(void)
139{
140}
141#else
142static void __init arm64_memory_present(void)
143{
144 struct memblock_region *reg;
145
146 for_each_memblock(memory, reg)
147 memory_present(0, memblock_region_memory_base_pfn(reg),
148 memblock_region_memory_end_pfn(reg));
149}
150#endif
151
152static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
153
154/*
155 * Limit the memory size that was specified via FDT.
156 */
157static int __init early_mem(char *p)
158{
159 if (!p)
160 return 1;
161
162 memory_limit = memparse(p, &p) & PAGE_MASK;
163 pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
164
165 return 0;
166}
167early_param("mem", early_mem);
168
169void __init arm64_memblock_init(void)
170{
171 const s64 linear_region_size = -(s64)PAGE_OFFSET;
172
173 /*
174 * Ensure that the linear region takes up exactly half of the kernel
175 * virtual address space. This way, we can distinguish a linear address
176 * from a kernel/module/vmalloc address by testing a single bit.
177 */
178 BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1));
179
180 /*
181 * Select a suitable value for the base of physical memory.
182 */
183 memstart_addr = round_down(memblock_start_of_DRAM(),
184 ARM64_MEMSTART_ALIGN);
185
186 /*
187 * Remove the memory that we will not be able to cover with the
188 * linear mapping. Take care not to clip the kernel which may be
189 * high in memory.
190 */
191 memblock_remove(max_t(u64, memstart_addr + linear_region_size, __pa(_end)),
192 ULLONG_MAX);
193 if (memblock_end_of_DRAM() > linear_region_size)
194 memblock_remove(0, memblock_end_of_DRAM() - linear_region_size);
195
196 /*
197 * Apply the memory limit if it was set. Since the kernel may be loaded
198 * high up in memory, add back the kernel region that must be accessible
199 * via the linear mapping.
200 */
201 if (memory_limit != (phys_addr_t)ULLONG_MAX) {
202 memblock_enforce_memory_limit(memory_limit);
203 memblock_add(__pa(_text), (u64)(_end - _text));
204 }
205
206 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
207 extern u16 memstart_offset_seed;
208 u64 range = linear_region_size -
209 (memblock_end_of_DRAM() - memblock_start_of_DRAM());
210
211 /*
212 * If the size of the linear region exceeds, by a sufficient
213 * margin, the size of the region that the available physical
214 * memory spans, randomize the linear region as well.
215 */
216 if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) {
217 range = range / ARM64_MEMSTART_ALIGN + 1;
218 memstart_addr -= ARM64_MEMSTART_ALIGN *
219 ((range * memstart_offset_seed) >> 16);
220 }
221 }
222
223 /*
224 * Register the kernel text, kernel data, initrd, and initial
225 * pagetables with memblock.
226 */
227 memblock_reserve(__pa(_text), _end - _text);
228#ifdef CONFIG_BLK_DEV_INITRD
229 if (initrd_start) {
230 memblock_reserve(initrd_start, initrd_end - initrd_start);
231
232 /* the generic initrd code expects virtual addresses */
233 initrd_start = __phys_to_virt(initrd_start);
234 initrd_end = __phys_to_virt(initrd_end);
235 }
236#endif
237
238 early_init_fdt_scan_reserved_mem();
239
240 /* 4GB maximum for 32-bit only capable devices */
241 if (IS_ENABLED(CONFIG_ZONE_DMA))
242 arm64_dma_phys_limit = max_zone_dma_phys();
243 else
244 arm64_dma_phys_limit = PHYS_MASK + 1;
245 dma_contiguous_reserve(arm64_dma_phys_limit);
246
247 memblock_allow_resize();
248 memblock_dump_all();
249}
250
251void __init bootmem_init(void)
252{
253 unsigned long min, max;
254
255 min = PFN_UP(memblock_start_of_DRAM());
256 max = PFN_DOWN(memblock_end_of_DRAM());
257
258 early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);
259
260 /*
261 * Sparsemem tries to allocate bootmem in memory_present(), so must be
262 * done after the fixed reservations.
263 */
264 arm64_memory_present();
265
266 sparse_init();
267 zone_sizes_init(min, max);
268
269 high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
270 max_pfn = max_low_pfn = max;
271}
272
273#ifndef CONFIG_SPARSEMEM_VMEMMAP
274static inline void free_memmap(unsigned long start_pfn, unsigned long end_pfn)
275{
276 struct page *start_pg, *end_pg;
277 unsigned long pg, pgend;
278
279 /*
280 * Convert start_pfn/end_pfn to a struct page pointer.
281 */
282 start_pg = pfn_to_page(start_pfn - 1) + 1;
283 end_pg = pfn_to_page(end_pfn - 1) + 1;
284
285 /*
286 * Convert to physical addresses, and round start upwards and end
287 * downwards.
288 */
289 pg = (unsigned long)PAGE_ALIGN(__pa(start_pg));
290 pgend = (unsigned long)__pa(end_pg) & PAGE_MASK;
291
292 /*
293 * If there are free pages between these, free the section of the
294 * memmap array.
295 */
296 if (pg < pgend)
297 free_bootmem(pg, pgend - pg);
298}
299
300/*
301 * The mem_map array can get very big. Free the unused area of the memory map.
302 */
303static void __init free_unused_memmap(void)
304{
305 unsigned long start, prev_end = 0;
306 struct memblock_region *reg;
307
308 for_each_memblock(memory, reg) {
309 start = __phys_to_pfn(reg->base);
310
311#ifdef CONFIG_SPARSEMEM
312 /*
313 * Take care not to free memmap entries that don't exist due
314 * to SPARSEMEM sections which aren't present.
315 */
316 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
317#endif
318 /*
319 * If we had a previous bank, and there is a space between the
320 * current bank and the previous, free it.
321 */
322 if (prev_end && prev_end < start)
323 free_memmap(prev_end, start);
324
325 /*
326 * Align up here since the VM subsystem insists that the
327 * memmap entries are valid from the bank end aligned to
328 * MAX_ORDER_NR_PAGES.
329 */
330 prev_end = ALIGN(__phys_to_pfn(reg->base + reg->size),
331 MAX_ORDER_NR_PAGES);
332 }
333
334#ifdef CONFIG_SPARSEMEM
335 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
336 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
337#endif
338}
339#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
340
341/*
342 * mem_init() marks the free areas in the mem_map and tells us how much memory
343 * is free. This is done after various parts of the system have claimed their
344 * memory after the kernel image.
345 */
346void __init mem_init(void)
347{
348 swiotlb_init(1);
349
350 set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
351
352#ifndef CONFIG_SPARSEMEM_VMEMMAP
353 free_unused_memmap();
354#endif
355 /* this will put all unused low memory onto the freelists */
356 free_all_bootmem();
357
358 mem_init_print_info(NULL);
359
360#define MLK(b, t) b, t, ((t) - (b)) >> 10
361#define MLM(b, t) b, t, ((t) - (b)) >> 20
362#define MLG(b, t) b, t, ((t) - (b)) >> 30
363#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
364
365 pr_notice("Virtual kernel memory layout:\n");
366#ifdef CONFIG_KASAN
367 pr_cont(" kasan : 0x%16lx - 0x%16lx (%6ld GB)\n",
368 MLG(KASAN_SHADOW_START, KASAN_SHADOW_END));
369#endif
370 pr_cont(" modules : 0x%16lx - 0x%16lx (%6ld MB)\n",
371 MLM(MODULES_VADDR, MODULES_END));
372 pr_cont(" vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n",
373 MLG(VMALLOC_START, VMALLOC_END));
374 pr_cont(" .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
375 " .rodata : 0x%p" " - 0x%p" " (%6ld KB)\n"
376 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
377 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
378 MLK_ROUNDUP(_text, __start_rodata),
379 MLK_ROUNDUP(__start_rodata, _etext),
380 MLK_ROUNDUP(__init_begin, __init_end),
381 MLK_ROUNDUP(_sdata, _edata));
382#ifdef CONFIG_SPARSEMEM_VMEMMAP
383 pr_cont(" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
384 " 0x%16lx - 0x%16lx (%6ld MB actual)\n",
385 MLG(VMEMMAP_START,
386 VMEMMAP_START + VMEMMAP_SIZE),
387 MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),
388 (unsigned long)virt_to_page(high_memory)));
389#endif
390 pr_cont(" fixed : 0x%16lx - 0x%16lx (%6ld KB)\n",
391 MLK(FIXADDR_START, FIXADDR_TOP));
392 pr_cont(" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n",
393 MLM(PCI_IO_START, PCI_IO_END));
394 pr_cont(" memory : 0x%16lx - 0x%16lx (%6ld MB)\n",
395 MLM(__phys_to_virt(memblock_start_of_DRAM()),
396 (unsigned long)high_memory));
397
398#undef MLK
399#undef MLM
400#undef MLK_ROUNDUP
401
402 /*
403 * Check boundaries twice: Some fundamental inconsistencies can be
404 * detected at build time already.
405 */
406#ifdef CONFIG_COMPAT
407 BUILD_BUG_ON(TASK_SIZE_32 > TASK_SIZE_64);
408#endif
409
410 if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
411 extern int sysctl_overcommit_memory;
412 /*
413 * On a machine this small we won't get anywhere without
414 * overcommit, so turn it on by default.
415 */
416 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
417 }
418}
419
420void free_initmem(void)
421{
422 free_initmem_default(0);
423 fixup_init();
424}
425
426#ifdef CONFIG_BLK_DEV_INITRD
427
428static int keep_initrd __initdata;
429
430void __init free_initrd_mem(unsigned long start, unsigned long end)
431{
432 if (!keep_initrd)
433 free_reserved_area((void *)start, (void *)end, 0, "initrd");
434}
435
436static int __init keepinitrd_setup(char *__unused)
437{
438 keep_initrd = 1;
439 return 1;
440}
441
442__setup("keepinitrd", keepinitrd_setup);
443#endif
444
445/*
446 * Dump out memory limit information on panic.
447 */
448static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p)
449{
450 if (memory_limit != (phys_addr_t)ULLONG_MAX) {
451 pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
452 } else {
453 pr_emerg("Memory Limit: none\n");
454 }
455 return 0;
456}
457
458static struct notifier_block mem_limit_notifier = {
459 .notifier_call = dump_mem_limit,
460};
461
462static int __init register_mem_limit_dumper(void)
463{
464 atomic_notifier_chain_register(&panic_notifier_list,
465 &mem_limit_notifier);
466 return 0;
467}
468__initcall(register_mem_limit_dumper);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/mm/init.c
4 *
5 * Copyright (C) 1995-2005 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9#include <linux/kernel.h>
10#include <linux/export.h>
11#include <linux/errno.h>
12#include <linux/swap.h>
13#include <linux/init.h>
14#include <linux/cache.h>
15#include <linux/mman.h>
16#include <linux/nodemask.h>
17#include <linux/initrd.h>
18#include <linux/gfp.h>
19#include <linux/memblock.h>
20#include <linux/sort.h>
21#include <linux/of.h>
22#include <linux/of_fdt.h>
23#include <linux/dma-direct.h>
24#include <linux/dma-map-ops.h>
25#include <linux/efi.h>
26#include <linux/swiotlb.h>
27#include <linux/vmalloc.h>
28#include <linux/mm.h>
29#include <linux/kexec.h>
30#include <linux/crash_dump.h>
31#include <linux/hugetlb.h>
32#include <linux/acpi_iort.h>
33
34#include <asm/boot.h>
35#include <asm/fixmap.h>
36#include <asm/kasan.h>
37#include <asm/kernel-pgtable.h>
38#include <asm/kvm_host.h>
39#include <asm/memory.h>
40#include <asm/numa.h>
41#include <asm/sections.h>
42#include <asm/setup.h>
43#include <linux/sizes.h>
44#include <asm/tlb.h>
45#include <asm/alternative.h>
46#include <asm/xen/swiotlb-xen.h>
47
48/*
49 * We need to be able to catch inadvertent references to memstart_addr
50 * that occur (potentially in generic code) before arm64_memblock_init()
51 * executes, which assigns it its actual value. So use a default value
52 * that cannot be mistaken for a real physical address.
53 */
54s64 memstart_addr __ro_after_init = -1;
55EXPORT_SYMBOL(memstart_addr);
56
57/*
58 * If the corresponding config options are enabled, we create both ZONE_DMA
59 * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory
60 * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4).
61 * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory,
62 * otherwise it is empty.
63 */
64phys_addr_t arm64_dma_phys_limit __ro_after_init;
65
66#ifdef CONFIG_KEXEC_CORE
67/*
68 * reserve_crashkernel() - reserves memory for crash kernel
69 *
70 * This function reserves memory area given in "crashkernel=" kernel command
71 * line parameter. The memory reserved is used by dump capture kernel when
72 * primary kernel is crashing.
73 */
74static void __init reserve_crashkernel(void)
75{
76 unsigned long long crash_base, crash_size;
77 int ret;
78
79 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
80 &crash_size, &crash_base);
81 /* no crashkernel= or invalid value specified */
82 if (ret || !crash_size)
83 return;
84
85 crash_size = PAGE_ALIGN(crash_size);
86
87 if (crash_base == 0) {
88 /* Current arm64 boot protocol requires 2MB alignment */
89 crash_base = memblock_find_in_range(0, arm64_dma_phys_limit,
90 crash_size, SZ_2M);
91 if (crash_base == 0) {
92 pr_warn("cannot allocate crashkernel (size:0x%llx)\n",
93 crash_size);
94 return;
95 }
96 } else {
97 /* User specifies base address explicitly. */
98 if (!memblock_is_region_memory(crash_base, crash_size)) {
99 pr_warn("cannot reserve crashkernel: region is not memory\n");
100 return;
101 }
102
103 if (memblock_is_region_reserved(crash_base, crash_size)) {
104 pr_warn("cannot reserve crashkernel: region overlaps reserved memory\n");
105 return;
106 }
107
108 if (!IS_ALIGNED(crash_base, SZ_2M)) {
109 pr_warn("cannot reserve crashkernel: base address is not 2MB aligned\n");
110 return;
111 }
112 }
113 memblock_reserve(crash_base, crash_size);
114
115 pr_info("crashkernel reserved: 0x%016llx - 0x%016llx (%lld MB)\n",
116 crash_base, crash_base + crash_size, crash_size >> 20);
117
118 crashk_res.start = crash_base;
119 crashk_res.end = crash_base + crash_size - 1;
120}
121#else
122static void __init reserve_crashkernel(void)
123{
124}
125#endif /* CONFIG_KEXEC_CORE */
126
127#ifdef CONFIG_CRASH_DUMP
128static int __init early_init_dt_scan_elfcorehdr(unsigned long node,
129 const char *uname, int depth, void *data)
130{
131 const __be32 *reg;
132 int len;
133
134 if (depth != 1 || strcmp(uname, "chosen") != 0)
135 return 0;
136
137 reg = of_get_flat_dt_prop(node, "linux,elfcorehdr", &len);
138 if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells)))
139 return 1;
140
141 elfcorehdr_addr = dt_mem_next_cell(dt_root_addr_cells, ®);
142 elfcorehdr_size = dt_mem_next_cell(dt_root_size_cells, ®);
143
144 return 1;
145}
146
147/*
148 * reserve_elfcorehdr() - reserves memory for elf core header
149 *
150 * This function reserves the memory occupied by an elf core header
151 * described in the device tree. This region contains all the
152 * information about primary kernel's core image and is used by a dump
153 * capture kernel to access the system memory on primary kernel.
154 */
155static void __init reserve_elfcorehdr(void)
156{
157 of_scan_flat_dt(early_init_dt_scan_elfcorehdr, NULL);
158
159 if (!elfcorehdr_size)
160 return;
161
162 if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) {
163 pr_warn("elfcorehdr is overlapped\n");
164 return;
165 }
166
167 memblock_reserve(elfcorehdr_addr, elfcorehdr_size);
168
169 pr_info("Reserving %lldKB of memory at 0x%llx for elfcorehdr\n",
170 elfcorehdr_size >> 10, elfcorehdr_addr);
171}
172#else
173static void __init reserve_elfcorehdr(void)
174{
175}
176#endif /* CONFIG_CRASH_DUMP */
177
178/*
179 * Return the maximum physical address for a zone accessible by the given bits
180 * limit. If DRAM starts above 32-bit, expand the zone to the maximum
181 * available memory, otherwise cap it at 32-bit.
182 */
183static phys_addr_t __init max_zone_phys(unsigned int zone_bits)
184{
185 phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits);
186 phys_addr_t phys_start = memblock_start_of_DRAM();
187
188 if (phys_start > U32_MAX)
189 zone_mask = PHYS_ADDR_MAX;
190 else if (phys_start > zone_mask)
191 zone_mask = U32_MAX;
192
193 return min(zone_mask, memblock_end_of_DRAM() - 1) + 1;
194}
195
196static void __init zone_sizes_init(unsigned long min, unsigned long max)
197{
198 unsigned long max_zone_pfns[MAX_NR_ZONES] = {0};
199 unsigned int __maybe_unused acpi_zone_dma_bits;
200 unsigned int __maybe_unused dt_zone_dma_bits;
201 phys_addr_t __maybe_unused dma32_phys_limit = max_zone_phys(32);
202
203#ifdef CONFIG_ZONE_DMA
204 acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address());
205 dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL));
206 zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits);
207 arm64_dma_phys_limit = max_zone_phys(zone_dma_bits);
208 max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit);
209#endif
210#ifdef CONFIG_ZONE_DMA32
211 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit);
212 if (!arm64_dma_phys_limit)
213 arm64_dma_phys_limit = dma32_phys_limit;
214#endif
215 if (!arm64_dma_phys_limit)
216 arm64_dma_phys_limit = PHYS_MASK + 1;
217 max_zone_pfns[ZONE_NORMAL] = max;
218
219 free_area_init(max_zone_pfns);
220}
221
222int pfn_valid(unsigned long pfn)
223{
224 phys_addr_t addr = PFN_PHYS(pfn);
225 struct mem_section *ms;
226
227 /*
228 * Ensure the upper PAGE_SHIFT bits are clear in the
229 * pfn. Else it might lead to false positives when
230 * some of the upper bits are set, but the lower bits
231 * match a valid pfn.
232 */
233 if (PHYS_PFN(addr) != pfn)
234 return 0;
235
236 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
237 return 0;
238
239 ms = __pfn_to_section(pfn);
240 if (!valid_section(ms))
241 return 0;
242
243 /*
244 * ZONE_DEVICE memory does not have the memblock entries.
245 * memblock_is_map_memory() check for ZONE_DEVICE based
246 * addresses will always fail. Even the normal hotplugged
247 * memory will never have MEMBLOCK_NOMAP flag set in their
248 * memblock entries. Skip memblock search for all non early
249 * memory sections covering all of hotplug memory including
250 * both normal and ZONE_DEVICE based.
251 */
252 if (!early_section(ms))
253 return pfn_section_valid(ms, pfn);
254
255 return memblock_is_memory(addr);
256}
257EXPORT_SYMBOL(pfn_valid);
258
259int pfn_is_map_memory(unsigned long pfn)
260{
261 phys_addr_t addr = PFN_PHYS(pfn);
262
263 /* avoid false positives for bogus PFNs, see comment in pfn_valid() */
264 if (PHYS_PFN(addr) != pfn)
265 return 0;
266
267 return memblock_is_map_memory(addr);
268}
269EXPORT_SYMBOL(pfn_is_map_memory);
270
271static phys_addr_t memory_limit = PHYS_ADDR_MAX;
272
273/*
274 * Limit the memory size that was specified via FDT.
275 */
276static int __init early_mem(char *p)
277{
278 if (!p)
279 return 1;
280
281 memory_limit = memparse(p, &p) & PAGE_MASK;
282 pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
283
284 return 0;
285}
286early_param("mem", early_mem);
287
288static int __init early_init_dt_scan_usablemem(unsigned long node,
289 const char *uname, int depth, void *data)
290{
291 struct memblock_region *usablemem = data;
292 const __be32 *reg;
293 int len;
294
295 if (depth != 1 || strcmp(uname, "chosen") != 0)
296 return 0;
297
298 reg = of_get_flat_dt_prop(node, "linux,usable-memory-range", &len);
299 if (!reg || (len < (dt_root_addr_cells + dt_root_size_cells)))
300 return 1;
301
302 usablemem->base = dt_mem_next_cell(dt_root_addr_cells, ®);
303 usablemem->size = dt_mem_next_cell(dt_root_size_cells, ®);
304
305 return 1;
306}
307
308static void __init fdt_enforce_memory_region(void)
309{
310 struct memblock_region reg = {
311 .size = 0,
312 };
313
314 of_scan_flat_dt(early_init_dt_scan_usablemem, ®);
315
316 if (reg.size)
317 memblock_cap_memory_range(reg.base, reg.size);
318}
319
320void __init arm64_memblock_init(void)
321{
322 s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
323
324 /*
325 * Corner case: 52-bit VA capable systems running KVM in nVHE mode may
326 * be limited in their ability to support a linear map that exceeds 51
327 * bits of VA space, depending on the placement of the ID map. Given
328 * that the placement of the ID map may be randomized, let's simply
329 * limit the kernel's linear map to 51 bits as well if we detect this
330 * configuration.
331 */
332 if (IS_ENABLED(CONFIG_KVM) && vabits_actual == 52 &&
333 is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
334 pr_info("Capping linear region to 51 bits for KVM in nVHE mode on LVA capable hardware.\n");
335 linear_region_size = min_t(u64, linear_region_size, BIT(51));
336 }
337
338 /* Handle linux,usable-memory-range property */
339 fdt_enforce_memory_region();
340
341 /* Remove memory above our supported physical address size */
342 memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX);
343
344 /*
345 * Select a suitable value for the base of physical memory.
346 */
347 memstart_addr = round_down(memblock_start_of_DRAM(),
348 ARM64_MEMSTART_ALIGN);
349
350 if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size)
351 pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n");
352
353 /*
354 * Remove the memory that we will not be able to cover with the
355 * linear mapping. Take care not to clip the kernel which may be
356 * high in memory.
357 */
358 memblock_remove(max_t(u64, memstart_addr + linear_region_size,
359 __pa_symbol(_end)), ULLONG_MAX);
360 if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) {
361 /* ensure that memstart_addr remains sufficiently aligned */
362 memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size,
363 ARM64_MEMSTART_ALIGN);
364 memblock_remove(0, memstart_addr);
365 }
366
367 /*
368 * If we are running with a 52-bit kernel VA config on a system that
369 * does not support it, we have to place the available physical
370 * memory in the 48-bit addressable part of the linear region, i.e.,
371 * we have to move it upward. Since memstart_addr represents the
372 * physical address of PAGE_OFFSET, we have to *subtract* from it.
373 */
374 if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52) && (vabits_actual != 52))
375 memstart_addr -= _PAGE_OFFSET(48) - _PAGE_OFFSET(52);
376
377 /*
378 * Apply the memory limit if it was set. Since the kernel may be loaded
379 * high up in memory, add back the kernel region that must be accessible
380 * via the linear mapping.
381 */
382 if (memory_limit != PHYS_ADDR_MAX) {
383 memblock_mem_limit_remove_map(memory_limit);
384 memblock_add(__pa_symbol(_text), (u64)(_end - _text));
385 }
386
387 if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) {
388 /*
389 * Add back the memory we just removed if it results in the
390 * initrd to become inaccessible via the linear mapping.
391 * Otherwise, this is a no-op
392 */
393 u64 base = phys_initrd_start & PAGE_MASK;
394 u64 size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base;
395
396 /*
397 * We can only add back the initrd memory if we don't end up
398 * with more memory than we can address via the linear mapping.
399 * It is up to the bootloader to position the kernel and the
400 * initrd reasonably close to each other (i.e., within 32 GB of
401 * each other) so that all granule/#levels combinations can
402 * always access both.
403 */
404 if (WARN(base < memblock_start_of_DRAM() ||
405 base + size > memblock_start_of_DRAM() +
406 linear_region_size,
407 "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) {
408 phys_initrd_size = 0;
409 } else {
410 memblock_remove(base, size); /* clear MEMBLOCK_ flags */
411 memblock_add(base, size);
412 memblock_reserve(base, size);
413 }
414 }
415
416 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
417 extern u16 memstart_offset_seed;
418 u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
419 int parange = cpuid_feature_extract_unsigned_field(
420 mmfr0, ID_AA64MMFR0_PARANGE_SHIFT);
421 s64 range = linear_region_size -
422 BIT(id_aa64mmfr0_parange_to_phys_shift(parange));
423
424 /*
425 * If the size of the linear region exceeds, by a sufficient
426 * margin, the size of the region that the physical memory can
427 * span, randomize the linear region as well.
428 */
429 if (memstart_offset_seed > 0 && range >= (s64)ARM64_MEMSTART_ALIGN) {
430 range /= ARM64_MEMSTART_ALIGN;
431 memstart_addr -= ARM64_MEMSTART_ALIGN *
432 ((range * memstart_offset_seed) >> 16);
433 }
434 }
435
436 /*
437 * Register the kernel text, kernel data, initrd, and initial
438 * pagetables with memblock.
439 */
440 memblock_reserve(__pa_symbol(_stext), _end - _stext);
441 if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) {
442 /* the generic initrd code expects virtual addresses */
443 initrd_start = __phys_to_virt(phys_initrd_start);
444 initrd_end = initrd_start + phys_initrd_size;
445 }
446
447 early_init_fdt_scan_reserved_mem();
448
449 reserve_elfcorehdr();
450
451 high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
452}
453
454void __init bootmem_init(void)
455{
456 unsigned long min, max;
457
458 min = PFN_UP(memblock_start_of_DRAM());
459 max = PFN_DOWN(memblock_end_of_DRAM());
460
461 early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);
462
463 max_pfn = max_low_pfn = max;
464 min_low_pfn = min;
465
466 arch_numa_init();
467
468 /*
469 * must be done after arch_numa_init() which calls numa_init() to
470 * initialize node_online_map that gets used in hugetlb_cma_reserve()
471 * while allocating required CMA size across online nodes.
472 */
473#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
474 arm64_hugetlb_cma_reserve();
475#endif
476
477 dma_pernuma_cma_reserve();
478
479 kvm_hyp_reserve();
480
481 /*
482 * sparse_init() tries to allocate memory from memblock, so must be
483 * done after the fixed reservations
484 */
485 sparse_init();
486 zone_sizes_init(min, max);
487
488 /*
489 * Reserve the CMA area after arm64_dma_phys_limit was initialised.
490 */
491 dma_contiguous_reserve(arm64_dma_phys_limit);
492
493 /*
494 * request_standard_resources() depends on crashkernel's memory being
495 * reserved, so do it here.
496 */
497 reserve_crashkernel();
498
499 memblock_dump_all();
500}
501
502/*
503 * mem_init() marks the free areas in the mem_map and tells us how much memory
504 * is free. This is done after various parts of the system have claimed their
505 * memory after the kernel image.
506 */
507void __init mem_init(void)
508{
509 if (swiotlb_force == SWIOTLB_FORCE ||
510 max_pfn > PFN_DOWN(arm64_dma_phys_limit))
511 swiotlb_init(1);
512 else if (!xen_swiotlb_detect())
513 swiotlb_force = SWIOTLB_NO_FORCE;
514
515 set_max_mapnr(max_pfn - PHYS_PFN_OFFSET);
516
517 /* this will put all unused low memory onto the freelists */
518 memblock_free_all();
519
520 /*
521 * Check boundaries twice: Some fundamental inconsistencies can be
522 * detected at build time already.
523 */
524#ifdef CONFIG_COMPAT
525 BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64);
526#endif
527
528 /*
529 * Selected page table levels should match when derived from
530 * scratch using the virtual address range and page size.
531 */
532 BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) !=
533 CONFIG_PGTABLE_LEVELS);
534
535 if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
536 extern int sysctl_overcommit_memory;
537 /*
538 * On a machine this small we won't get anywhere without
539 * overcommit, so turn it on by default.
540 */
541 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
542 }
543}
544
545void free_initmem(void)
546{
547 free_reserved_area(lm_alias(__init_begin),
548 lm_alias(__init_end),
549 POISON_FREE_INITMEM, "unused kernel");
550 /*
551 * Unmap the __init region but leave the VM area in place. This
552 * prevents the region from being reused for kernel modules, which
553 * is not supported by kallsyms.
554 */
555 vunmap_range((u64)__init_begin, (u64)__init_end);
556}
557
558void dump_mem_limit(void)
559{
560 if (memory_limit != PHYS_ADDR_MAX) {
561 pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
562 } else {
563 pr_emerg("Memory Limit: none\n");
564 }
565}