Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Support for Kernel relocation at boot time
4 *
5 * Copyright (C) 2023 Loongson Technology Corporation Limited
6 */
7
8#include <linux/elf.h>
9#include <linux/kernel.h>
10#include <linux/printk.h>
11#include <linux/panic_notifier.h>
12#include <linux/start_kernel.h>
13#include <asm/bootinfo.h>
14#include <asm/early_ioremap.h>
15#include <asm/inst.h>
16#include <asm/sections.h>
17#include <asm/setup.h>
18
19#define RELOCATED(x) ((void *)((long)x + reloc_offset))
20#define RELOCATED_KASLR(x) ((void *)((long)x + random_offset))
21
22static unsigned long reloc_offset;
23
24static inline void __init relocate_relative(void)
25{
26 Elf64_Rela *rela, *rela_end;
27 rela = (Elf64_Rela *)&__rela_dyn_begin;
28 rela_end = (Elf64_Rela *)&__rela_dyn_end;
29
30 for ( ; rela < rela_end; rela++) {
31 Elf64_Addr addr = rela->r_offset;
32 Elf64_Addr relocated_addr = rela->r_addend;
33
34 if (rela->r_info != R_LARCH_RELATIVE)
35 continue;
36
37 if (relocated_addr >= VMLINUX_LOAD_ADDRESS)
38 relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr);
39
40 *(Elf64_Addr *)RELOCATED(addr) = relocated_addr;
41 }
42}
43
44static inline void __init relocate_absolute(long random_offset)
45{
46 void *begin, *end;
47 struct rela_la_abs *p;
48
49 begin = RELOCATED_KASLR(&__la_abs_begin);
50 end = RELOCATED_KASLR(&__la_abs_end);
51
52 for (p = begin; (void *)p < end; p++) {
53 long v = p->symvalue;
54 uint32_t lu12iw, ori, lu32id, lu52id;
55 union loongarch_instruction *insn = (void *)p->pc;
56
57 lu12iw = (v >> 12) & 0xfffff;
58 ori = v & 0xfff;
59 lu32id = (v >> 32) & 0xfffff;
60 lu52id = v >> 52;
61
62 insn[0].reg1i20_format.immediate = lu12iw;
63 insn[1].reg2i12_format.immediate = ori;
64 insn[2].reg1i20_format.immediate = lu32id;
65 insn[3].reg2i12_format.immediate = lu52id;
66 }
67}
68
69#ifdef CONFIG_RANDOMIZE_BASE
70static inline __init unsigned long rotate_xor(unsigned long hash,
71 const void *area, size_t size)
72{
73 size_t i, diff;
74 const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash));
75
76 diff = (void *)ptr - area;
77 if (size < diff + sizeof(hash))
78 return hash;
79
80 size = ALIGN_DOWN(size - diff, sizeof(hash));
81
82 for (i = 0; i < size / sizeof(hash); i++) {
83 /* Rotate by odd number of bits and XOR. */
84 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
85 hash ^= ptr[i];
86 }
87
88 return hash;
89}
90
91static inline __init unsigned long get_random_boot(void)
92{
93 unsigned long hash = 0;
94 unsigned long entropy = random_get_entropy();
95
96 /* Attempt to create a simple but unpredictable starting entropy. */
97 hash = rotate_xor(hash, linux_banner, strlen(linux_banner));
98
99 /* Add in any runtime entropy we can get */
100 hash = rotate_xor(hash, &entropy, sizeof(entropy));
101
102 return hash;
103}
104
105static int __init nokaslr(char *p)
106{
107 pr_info("KASLR is disabled.\n");
108
109 return 0; /* Print a notice and silence the boot warning */
110}
111early_param("nokaslr", nokaslr);
112
113static inline __init bool kaslr_disabled(void)
114{
115 char *str;
116 const char *builtin_cmdline = CONFIG_CMDLINE;
117
118 str = strstr(builtin_cmdline, "nokaslr");
119 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
120 return true;
121
122 str = strstr(boot_command_line, "nokaslr");
123 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
124 return true;
125
126 return false;
127}
128
129/* Choose a new address for the kernel */
130static inline void __init *determine_relocation_address(void)
131{
132 unsigned long kernel_length;
133 unsigned long random_offset;
134 void *destination = _text;
135
136 if (kaslr_disabled())
137 return destination;
138
139 kernel_length = (long)_end - (long)_text;
140
141 random_offset = get_random_boot() << 16;
142 random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1);
143 if (random_offset < kernel_length)
144 random_offset += ALIGN(kernel_length, 0xffff);
145
146 return RELOCATED_KASLR(destination);
147}
148
149static inline int __init relocation_addr_valid(void *location_new)
150{
151 if ((unsigned long)location_new & 0x00000ffff)
152 return 0; /* Inappropriately aligned new location */
153
154 if ((unsigned long)location_new < (unsigned long)_end)
155 return 0; /* New location overlaps original kernel */
156
157 return 1;
158}
159#endif
160
161static inline void __init update_reloc_offset(unsigned long *addr, long random_offset)
162{
163 unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr);
164
165 *new_addr = (unsigned long)reloc_offset;
166}
167
168unsigned long __init relocate_kernel(void)
169{
170 unsigned long kernel_length;
171 unsigned long random_offset = 0;
172 void *location_new = _text; /* Default to original kernel start */
173 char *cmdline = early_ioremap(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */
174
175 strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE);
176
177#ifdef CONFIG_RANDOMIZE_BASE
178 location_new = determine_relocation_address();
179
180 /* Sanity check relocation address */
181 if (relocation_addr_valid(location_new))
182 random_offset = (unsigned long)location_new - (unsigned long)(_text);
183#endif
184 reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS;
185
186 if (random_offset) {
187 kernel_length = (long)(_end) - (long)(_text);
188
189 /* Copy the kernel to it's new location */
190 memcpy(location_new, _text, kernel_length);
191
192 /* Sync the caches ready for execution of new kernel */
193 __asm__ __volatile__ (
194 "ibar 0 \t\n"
195 "dbar 0 \t\n"
196 ::: "memory");
197
198 reloc_offset += random_offset;
199
200 /* The current thread is now within the relocated kernel */
201 __current_thread_info = RELOCATED_KASLR(__current_thread_info);
202
203 update_reloc_offset(&reloc_offset, random_offset);
204 }
205
206 if (reloc_offset)
207 relocate_relative();
208
209 relocate_absolute(random_offset);
210
211 return random_offset;
212}
213
214/*
215 * Show relocation information on panic.
216 */
217static void show_kernel_relocation(const char *level)
218{
219 if (reloc_offset > 0) {
220 printk(level);
221 pr_cont("Kernel relocated by 0x%lx\n", reloc_offset);
222 pr_cont(" .text @ 0x%px\n", _text);
223 pr_cont(" .data @ 0x%px\n", _sdata);
224 pr_cont(" .bss @ 0x%px\n", __bss_start);
225 }
226}
227
228static int kernel_location_notifier_fn(struct notifier_block *self,
229 unsigned long v, void *p)
230{
231 show_kernel_relocation(KERN_EMERG);
232 return NOTIFY_DONE;
233}
234
235static struct notifier_block kernel_location_notifier = {
236 .notifier_call = kernel_location_notifier_fn
237};
238
239static int __init register_kernel_offset_dumper(void)
240{
241 atomic_notifier_chain_register(&panic_notifier_list,
242 &kernel_location_notifier);
243 return 0;
244}
245
246arch_initcall(register_kernel_offset_dumper);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Support for Kernel relocation at boot time
4 *
5 * Copyright (C) 2023 Loongson Technology Corporation Limited
6 */
7
8#include <linux/elf.h>
9#include <linux/kernel.h>
10#include <linux/printk.h>
11#include <linux/panic_notifier.h>
12#include <linux/start_kernel.h>
13#include <asm/bootinfo.h>
14#include <asm/early_ioremap.h>
15#include <asm/inst.h>
16#include <asm/io.h>
17#include <asm/sections.h>
18#include <asm/setup.h>
19
20#define RELOCATED(x) ((void *)((long)x + reloc_offset))
21#define RELOCATED_KASLR(x) ((void *)((long)x + random_offset))
22
23static unsigned long reloc_offset;
24
25static inline void __init relocate_relative(void)
26{
27 Elf64_Rela *rela, *rela_end;
28 rela = (Elf64_Rela *)&__rela_dyn_begin;
29 rela_end = (Elf64_Rela *)&__rela_dyn_end;
30
31 for ( ; rela < rela_end; rela++) {
32 Elf64_Addr addr = rela->r_offset;
33 Elf64_Addr relocated_addr = rela->r_addend;
34
35 if (rela->r_info != R_LARCH_RELATIVE)
36 continue;
37
38 relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr);
39 *(Elf64_Addr *)RELOCATED(addr) = relocated_addr;
40 }
41
42#ifdef CONFIG_RELR
43 u64 *addr = NULL;
44 u64 *relr = (u64 *)&__relr_dyn_begin;
45 u64 *relr_end = (u64 *)&__relr_dyn_end;
46
47 for ( ; relr < relr_end; relr++) {
48 if ((*relr & 1) == 0) {
49 addr = (u64 *)(*relr + reloc_offset);
50 *addr++ += reloc_offset;
51 } else {
52 for (u64 *p = addr, r = *relr >> 1; r; p++, r >>= 1)
53 if (r & 1)
54 *p += reloc_offset;
55 addr += 63;
56 }
57 }
58#endif
59}
60
61static inline void __init relocate_absolute(long random_offset)
62{
63 void *begin, *end;
64 struct rela_la_abs *p;
65
66 begin = RELOCATED_KASLR(&__la_abs_begin);
67 end = RELOCATED_KASLR(&__la_abs_end);
68
69 for (p = begin; (void *)p < end; p++) {
70 long v = p->symvalue;
71 uint32_t lu12iw, ori, lu32id, lu52id;
72 union loongarch_instruction *insn = (void *)p->pc;
73
74 lu12iw = (v >> 12) & 0xfffff;
75 ori = v & 0xfff;
76 lu32id = (v >> 32) & 0xfffff;
77 lu52id = v >> 52;
78
79 insn[0].reg1i20_format.immediate = lu12iw;
80 insn[1].reg2i12_format.immediate = ori;
81 insn[2].reg1i20_format.immediate = lu32id;
82 insn[3].reg2i12_format.immediate = lu52id;
83 }
84}
85
86#ifdef CONFIG_RANDOMIZE_BASE
87static inline __init unsigned long rotate_xor(unsigned long hash,
88 const void *area, size_t size)
89{
90 size_t i, diff;
91 const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash));
92
93 diff = (void *)ptr - area;
94 if (size < diff + sizeof(hash))
95 return hash;
96
97 size = ALIGN_DOWN(size - diff, sizeof(hash));
98
99 for (i = 0; i < size / sizeof(hash); i++) {
100 /* Rotate by odd number of bits and XOR. */
101 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
102 hash ^= ptr[i];
103 }
104
105 return hash;
106}
107
108static inline __init unsigned long get_random_boot(void)
109{
110 unsigned long hash = 0;
111 unsigned long entropy = random_get_entropy();
112
113 /* Attempt to create a simple but unpredictable starting entropy. */
114 hash = rotate_xor(hash, linux_banner, strlen(linux_banner));
115
116 /* Add in any runtime entropy we can get */
117 hash = rotate_xor(hash, &entropy, sizeof(entropy));
118
119 return hash;
120}
121
122static int __init nokaslr(char *p)
123{
124 pr_info("KASLR is disabled.\n");
125
126 return 0; /* Print a notice and silence the boot warning */
127}
128early_param("nokaslr", nokaslr);
129
130static inline __init bool kaslr_disabled(void)
131{
132 char *str;
133 const char *builtin_cmdline = CONFIG_CMDLINE;
134
135 str = strstr(builtin_cmdline, "nokaslr");
136 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
137 return true;
138
139 str = strstr(boot_command_line, "nokaslr");
140 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
141 return true;
142
143#ifdef CONFIG_HIBERNATION
144 str = strstr(builtin_cmdline, "nohibernate");
145 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
146 return false;
147
148 str = strstr(boot_command_line, "nohibernate");
149 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
150 return false;
151
152 str = strstr(builtin_cmdline, "noresume");
153 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
154 return false;
155
156 str = strstr(boot_command_line, "noresume");
157 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
158 return false;
159
160 str = strstr(builtin_cmdline, "resume=");
161 if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' '))
162 return true;
163
164 str = strstr(boot_command_line, "resume=");
165 if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' '))
166 return true;
167#endif
168
169 return false;
170}
171
172/* Choose a new address for the kernel */
173static inline void __init *determine_relocation_address(void)
174{
175 unsigned long kernel_length;
176 unsigned long random_offset;
177 void *destination = _text;
178
179 if (kaslr_disabled())
180 return destination;
181
182 kernel_length = (long)_end - (long)_text;
183
184 random_offset = get_random_boot() << 16;
185 random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1);
186 if (random_offset < kernel_length)
187 random_offset += ALIGN(kernel_length, 0xffff);
188
189 return RELOCATED_KASLR(destination);
190}
191
192static inline int __init relocation_addr_valid(void *location_new)
193{
194 if ((unsigned long)location_new & 0x00000ffff)
195 return 0; /* Inappropriately aligned new location */
196
197 if ((unsigned long)location_new < (unsigned long)_end)
198 return 0; /* New location overlaps original kernel */
199
200 return 1;
201}
202#endif
203
204static inline void __init update_reloc_offset(unsigned long *addr, long random_offset)
205{
206 unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr);
207
208 *new_addr = (unsigned long)reloc_offset;
209}
210
211unsigned long __init relocate_kernel(void)
212{
213 unsigned long kernel_length;
214 unsigned long random_offset = 0;
215 void *location_new = _text; /* Default to original kernel start */
216 char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */
217
218 strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE);
219
220#ifdef CONFIG_RANDOMIZE_BASE
221 location_new = determine_relocation_address();
222
223 /* Sanity check relocation address */
224 if (relocation_addr_valid(location_new))
225 random_offset = (unsigned long)location_new - (unsigned long)(_text);
226#endif
227 reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS;
228 early_memunmap(cmdline, COMMAND_LINE_SIZE);
229
230 if (random_offset) {
231 kernel_length = (long)(_end) - (long)(_text);
232
233 /* Copy the kernel to it's new location */
234 memcpy(location_new, _text, kernel_length);
235
236 /* Sync the caches ready for execution of new kernel */
237 __asm__ __volatile__ (
238 "ibar 0 \t\n"
239 "dbar 0 \t\n"
240 ::: "memory");
241
242 reloc_offset += random_offset;
243
244 /* The current thread is now within the relocated kernel */
245 __current_thread_info = RELOCATED_KASLR(__current_thread_info);
246
247 update_reloc_offset(&reloc_offset, random_offset);
248 }
249
250 if (reloc_offset)
251 relocate_relative();
252
253 relocate_absolute(random_offset);
254
255 return random_offset;
256}
257
258/*
259 * Show relocation information on panic.
260 */
261static void show_kernel_relocation(const char *level)
262{
263 if (reloc_offset > 0) {
264 printk(level);
265 pr_cont("Kernel relocated by 0x%lx\n", reloc_offset);
266 pr_cont(" .text @ 0x%px\n", _text);
267 pr_cont(" .data @ 0x%px\n", _sdata);
268 pr_cont(" .bss @ 0x%px\n", __bss_start);
269 }
270}
271
272static int kernel_location_notifier_fn(struct notifier_block *self,
273 unsigned long v, void *p)
274{
275 show_kernel_relocation(KERN_EMERG);
276 return NOTIFY_DONE;
277}
278
279static struct notifier_block kernel_location_notifier = {
280 .notifier_call = kernel_location_notifier_fn
281};
282
283static int __init register_kernel_offset_dumper(void)
284{
285 atomic_notifier_chain_register(&panic_notifier_list,
286 &kernel_location_notifier);
287 return 0;
288}
289
290arch_initcall(register_kernel_offset_dumper);