Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kasan.h>
3#include <linux/sched/task.h>
4#include <linux/memblock.h>
5#include <linux/pgtable.h>
6#include <asm/pgalloc.h>
7#include <asm/kasan.h>
8#include <asm/mem_detect.h>
9#include <asm/processor.h>
10#include <asm/sclp.h>
11#include <asm/facility.h>
12#include <asm/sections.h>
13#include <asm/setup.h>
14#include <asm/uv.h>
15
16static unsigned long segment_pos __initdata;
17static unsigned long segment_low __initdata;
18static unsigned long pgalloc_pos __initdata;
19static unsigned long pgalloc_low __initdata;
20static unsigned long pgalloc_freeable __initdata;
21static bool has_edat __initdata;
22static bool has_nx __initdata;
23
24#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
25
26static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
27
28static void __init kasan_early_panic(const char *reason)
29{
30 sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
31 sclp_early_printk(reason);
32 disabled_wait();
33}
34
35static void * __init kasan_early_alloc_segment(void)
36{
37 segment_pos -= _SEGMENT_SIZE;
38
39 if (segment_pos < segment_low)
40 kasan_early_panic("out of memory during initialisation\n");
41
42 return (void *)segment_pos;
43}
44
45static void * __init kasan_early_alloc_pages(unsigned int order)
46{
47 pgalloc_pos -= (PAGE_SIZE << order);
48
49 if (pgalloc_pos < pgalloc_low)
50 kasan_early_panic("out of memory during initialisation\n");
51
52 return (void *)pgalloc_pos;
53}
54
55static void * __init kasan_early_crst_alloc(unsigned long val)
56{
57 unsigned long *table;
58
59 table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
60 if (table)
61 crst_table_init(table, val);
62 return table;
63}
64
65static pte_t * __init kasan_early_pte_alloc(void)
66{
67 static void *pte_leftover;
68 pte_t *pte;
69
70 BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
71
72 if (!pte_leftover) {
73 pte_leftover = kasan_early_alloc_pages(0);
74 pte = pte_leftover + _PAGE_TABLE_SIZE;
75 } else {
76 pte = pte_leftover;
77 pte_leftover = NULL;
78 }
79 memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
80 return pte;
81}
82
83enum populate_mode {
84 POPULATE_ONE2ONE,
85 POPULATE_MAP,
86 POPULATE_ZERO_SHADOW,
87 POPULATE_SHALLOW
88};
89static void __init kasan_early_pgtable_populate(unsigned long address,
90 unsigned long end,
91 enum populate_mode mode)
92{
93 unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
94 pgd_t *pg_dir;
95 p4d_t *p4_dir;
96 pud_t *pu_dir;
97 pmd_t *pm_dir;
98 pte_t *pt_dir;
99
100 pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
101 if (!has_nx)
102 pgt_prot_zero &= ~_PAGE_NOEXEC;
103 pgt_prot = pgprot_val(PAGE_KERNEL);
104 sgt_prot = pgprot_val(SEGMENT_KERNEL);
105 if (!has_nx || mode == POPULATE_ONE2ONE) {
106 pgt_prot &= ~_PAGE_NOEXEC;
107 sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
108 }
109
110 /*
111 * The first 1MB of 1:1 mapping is mapped with 4KB pages
112 */
113 while (address < end) {
114 pg_dir = pgd_offset_k(address);
115 if (pgd_none(*pg_dir)) {
116 if (mode == POPULATE_ZERO_SHADOW &&
117 IS_ALIGNED(address, PGDIR_SIZE) &&
118 end - address >= PGDIR_SIZE) {
119 pgd_populate(&init_mm, pg_dir,
120 kasan_early_shadow_p4d);
121 address = (address + PGDIR_SIZE) & PGDIR_MASK;
122 continue;
123 }
124 p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
125 pgd_populate(&init_mm, pg_dir, p4_dir);
126 }
127
128 if (mode == POPULATE_SHALLOW) {
129 address = (address + P4D_SIZE) & P4D_MASK;
130 continue;
131 }
132
133 p4_dir = p4d_offset(pg_dir, address);
134 if (p4d_none(*p4_dir)) {
135 if (mode == POPULATE_ZERO_SHADOW &&
136 IS_ALIGNED(address, P4D_SIZE) &&
137 end - address >= P4D_SIZE) {
138 p4d_populate(&init_mm, p4_dir,
139 kasan_early_shadow_pud);
140 address = (address + P4D_SIZE) & P4D_MASK;
141 continue;
142 }
143 pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
144 p4d_populate(&init_mm, p4_dir, pu_dir);
145 }
146
147 pu_dir = pud_offset(p4_dir, address);
148 if (pud_none(*pu_dir)) {
149 if (mode == POPULATE_ZERO_SHADOW &&
150 IS_ALIGNED(address, PUD_SIZE) &&
151 end - address >= PUD_SIZE) {
152 pud_populate(&init_mm, pu_dir,
153 kasan_early_shadow_pmd);
154 address = (address + PUD_SIZE) & PUD_MASK;
155 continue;
156 }
157 pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
158 pud_populate(&init_mm, pu_dir, pm_dir);
159 }
160
161 pm_dir = pmd_offset(pu_dir, address);
162 if (pmd_none(*pm_dir)) {
163 if (IS_ALIGNED(address, PMD_SIZE) &&
164 end - address >= PMD_SIZE) {
165 if (mode == POPULATE_ZERO_SHADOW) {
166 pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
167 address = (address + PMD_SIZE) & PMD_MASK;
168 continue;
169 } else if (has_edat && address) {
170 void *page;
171
172 if (mode == POPULATE_ONE2ONE) {
173 page = (void *)address;
174 } else {
175 page = kasan_early_alloc_segment();
176 memset(page, 0, _SEGMENT_SIZE);
177 }
178 set_pmd(pm_dir, __pmd(__pa(page) | sgt_prot));
179 address = (address + PMD_SIZE) & PMD_MASK;
180 continue;
181 }
182 }
183 pt_dir = kasan_early_pte_alloc();
184 pmd_populate(&init_mm, pm_dir, pt_dir);
185 } else if (pmd_large(*pm_dir)) {
186 address = (address + PMD_SIZE) & PMD_MASK;
187 continue;
188 }
189
190 pt_dir = pte_offset_kernel(pm_dir, address);
191 if (pte_none(*pt_dir)) {
192 void *page;
193
194 switch (mode) {
195 case POPULATE_ONE2ONE:
196 page = (void *)address;
197 set_pte(pt_dir, __pte(__pa(page) | pgt_prot));
198 break;
199 case POPULATE_MAP:
200 page = kasan_early_alloc_pages(0);
201 memset(page, 0, PAGE_SIZE);
202 set_pte(pt_dir, __pte(__pa(page) | pgt_prot));
203 break;
204 case POPULATE_ZERO_SHADOW:
205 page = kasan_early_shadow_page;
206 set_pte(pt_dir, __pte(__pa(page) | pgt_prot_zero));
207 break;
208 case POPULATE_SHALLOW:
209 /* should never happen */
210 break;
211 }
212 }
213 address += PAGE_SIZE;
214 }
215}
216
217static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
218{
219 unsigned long asce_bits;
220
221 asce_bits = asce_type | _ASCE_TABLE_LENGTH;
222 S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
223 S390_lowcore.user_asce = S390_lowcore.kernel_asce;
224
225 __ctl_load(S390_lowcore.kernel_asce, 1, 1);
226 __ctl_load(S390_lowcore.kernel_asce, 7, 7);
227 __ctl_load(S390_lowcore.kernel_asce, 13, 13);
228}
229
230static void __init kasan_enable_dat(void)
231{
232 psw_t psw;
233
234 psw.mask = __extract_psw();
235 psw_bits(psw).dat = 1;
236 psw_bits(psw).as = PSW_BITS_AS_HOME;
237 __load_psw_mask(psw.mask);
238}
239
240static void __init kasan_early_detect_facilities(void)
241{
242 if (test_facility(8)) {
243 has_edat = true;
244 __ctl_set_bit(0, 23);
245 }
246 if (!noexec_disabled && test_facility(130)) {
247 has_nx = true;
248 __ctl_set_bit(0, 20);
249 }
250}
251
252void __init kasan_early_init(void)
253{
254 unsigned long shadow_alloc_size;
255 unsigned long initrd_end;
256 unsigned long memsize;
257 unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
258 pte_t pte_z;
259 pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
260 pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
261 p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
262
263 kasan_early_detect_facilities();
264 if (!has_nx)
265 pgt_prot &= ~_PAGE_NOEXEC;
266 pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);
267
268 memsize = get_mem_detect_end();
269 if (!memsize)
270 kasan_early_panic("cannot detect physical memory size\n");
271 /*
272 * Kasan currently supports standby memory but only if it follows
273 * online memory (default allocation), i.e. no memory holes.
274 * - memsize represents end of online memory
275 * - ident_map_size represents online + standby and memory limits
276 * accounted.
277 * Kasan maps "memsize" right away.
278 * [0, memsize] - as identity mapping
279 * [__sha(0), __sha(memsize)] - shadow memory for identity mapping
280 * The rest [memsize, ident_map_size] if memsize < ident_map_size
281 * could be mapped/unmapped dynamically later during memory hotplug.
282 */
283 memsize = min(memsize, ident_map_size);
284
285 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
286 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
287 crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
288
289 /* init kasan zero shadow */
290 crst_table_init((unsigned long *)kasan_early_shadow_p4d,
291 p4d_val(p4d_z));
292 crst_table_init((unsigned long *)kasan_early_shadow_pud,
293 pud_val(pud_z));
294 crst_table_init((unsigned long *)kasan_early_shadow_pmd,
295 pmd_val(pmd_z));
296 memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
297
298 shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
299 pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
300 if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
301 initrd_end =
302 round_up(initrd_data.start + initrd_data.size, _SEGMENT_SIZE);
303 pgalloc_low = max(pgalloc_low, initrd_end);
304 }
305
306 if (pgalloc_low + shadow_alloc_size > memsize)
307 kasan_early_panic("out of memory during initialisation\n");
308
309 if (has_edat) {
310 segment_pos = round_down(memsize, _SEGMENT_SIZE);
311 segment_low = segment_pos - shadow_alloc_size;
312 pgalloc_pos = segment_low;
313 } else {
314 pgalloc_pos = memsize;
315 }
316 init_mm.pgd = early_pg_dir;
317 /*
318 * Current memory layout:
319 * +- 0 -------------+ +- shadow start -+
320 * | 1:1 ram mapping | /| 1/8 ram |
321 * | | / | |
322 * +- end of ram ----+ / +----------------+
323 * | ... gap ... | / | |
324 * | |/ | kasan |
325 * +- shadow start --+ | zero |
326 * | 1/8 addr space | | page |
327 * +- shadow end -+ | mapping |
328 * | ... gap ... |\ | (untracked) |
329 * +- vmalloc area -+ \ | |
330 * | vmalloc_size | \ | |
331 * +- modules vaddr -+ \ +----------------+
332 * | 2Gb | \| unmapped | allocated per module
333 * +-----------------+ +- shadow end ---+
334 *
335 * Current memory layout (KASAN_VMALLOC):
336 * +- 0 -------------+ +- shadow start -+
337 * | 1:1 ram mapping | /| 1/8 ram |
338 * | | / | |
339 * +- end of ram ----+ / +----------------+
340 * | ... gap ... | / | kasan |
341 * | |/ | zero |
342 * +- shadow start --+ | page |
343 * | 1/8 addr space | | mapping |
344 * +- shadow end -+ | (untracked) |
345 * | ... gap ... |\ | |
346 * +- vmalloc area -+ \ +- vmalloc area -+
347 * | vmalloc_size | \ |shallow populate|
348 * +- modules vaddr -+ \ +- modules area -+
349 * | 2Gb | \|shallow populate|
350 * +-----------------+ +- shadow end ---+
351 */
352 /* populate kasan shadow (for identity mapping and zero page mapping) */
353 kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
354 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
355 /* shallowly populate kasan shadow for vmalloc and modules */
356 kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
357 POPULATE_SHALLOW);
358 }
359 /* populate kasan shadow for untracked memory */
360 kasan_early_pgtable_populate(__sha(ident_map_size),
361 IS_ENABLED(CONFIG_KASAN_VMALLOC) ?
362 __sha(VMALLOC_START) :
363 __sha(MODULES_VADDR),
364 POPULATE_ZERO_SHADOW);
365 kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
366 POPULATE_ZERO_SHADOW);
367 /* memory allocated for identity mapping structs will be freed later */
368 pgalloc_freeable = pgalloc_pos;
369 /* populate identity mapping */
370 kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
371 kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
372 kasan_enable_dat();
373 /* enable kasan */
374 init_task.kasan_depth = 0;
375 memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
376 sclp_early_printk("KernelAddressSanitizer initialized\n");
377}
378
379void __init kasan_copy_shadow_mapping(void)
380{
381 /*
382 * At this point we are still running on early pages setup early_pg_dir,
383 * while swapper_pg_dir has just been initialized with identity mapping.
384 * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
385 */
386
387 pgd_t *pg_dir_src;
388 pgd_t *pg_dir_dst;
389 p4d_t *p4_dir_src;
390 p4d_t *p4_dir_dst;
391
392 pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
393 pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
394 p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
395 p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
396 memcpy(p4_dir_dst, p4_dir_src,
397 (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
398}
399
400void __init kasan_free_early_identity(void)
401{
402 memblock_phys_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
403}