1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Load ELF vmlinux file for the kexec_file_load syscall.
  4 *
  5 * Copyright (C) 2021 Huawei Technologies Co, Ltd.
  6 *
  7 * Author: Liao Chang (liaochang1@huawei.com)
  8 *
  9 * Based on kexec-tools' kexec-elf-riscv.c, heavily modified
 10 * for kernel.
 11 */
 12
 13#define pr_fmt(fmt)	"kexec_image: " fmt
 14
 15#include <linux/elf.h>
 16#include <linux/kexec.h>
 17#include <linux/slab.h>
 18#include <linux/of.h>
 19#include <linux/libfdt.h>
 20#include <linux/types.h>
 21#include <linux/memblock.h>
 22#include <asm/setup.h>
 23
 24int arch_kimage_file_post_load_cleanup(struct kimage *image)
 25{
 26	kvfree(image->arch.fdt);
 27	image->arch.fdt = NULL;
 28
 29	vfree(image->elf_headers);
 30	image->elf_headers = NULL;
 31	image->elf_headers_sz = 0;
 32
 33	return kexec_image_post_load_cleanup_default(image);
 34}
 35
 36static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr,
 37				struct kexec_elf_info *elf_info, unsigned long old_pbase,
 38				unsigned long new_pbase)
 39{
 40	int i;
 41	int ret = 0;
 42	size_t size;
 43	struct kexec_buf kbuf;
 44	const struct elf_phdr *phdr;
 45
 46	kbuf.image = image;
 47
 48	for (i = 0; i < ehdr->e_phnum; i++) {
 49		phdr = &elf_info->proghdrs[i];
 50		if (phdr->p_type != PT_LOAD)
 51			continue;
 52
 53		size = phdr->p_filesz;
 54		if (size > phdr->p_memsz)
 55			size = phdr->p_memsz;
 56
 57		kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
 58		kbuf.bufsz = size;
 59		kbuf.buf_align = phdr->p_align;
 60		kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
 61		kbuf.memsz = phdr->p_memsz;
 62		kbuf.top_down = false;
 63		ret = kexec_add_buffer(&kbuf);
 64		if (ret)
 65			break;
 66	}
 67
 68	return ret;
 69}
 70
 71/*
 72 * Go through the available phsyical memory regions and find one that hold
 73 * an image of the specified size.
 74 */
 75static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
 76			  struct elfhdr *ehdr, struct kexec_elf_info *elf_info,
 77			  unsigned long *old_pbase, unsigned long *new_pbase)
 78{
 79	int i;
 80	int ret;
 81	struct kexec_buf kbuf;
 82	const struct elf_phdr *phdr;
 83	unsigned long lowest_paddr = ULONG_MAX;
 84	unsigned long lowest_vaddr = ULONG_MAX;
 85
 86	for (i = 0; i < ehdr->e_phnum; i++) {
 87		phdr = &elf_info->proghdrs[i];
 88		if (phdr->p_type != PT_LOAD)
 89			continue;
 90
 91		if (lowest_paddr > phdr->p_paddr)
 92			lowest_paddr = phdr->p_paddr;
 93
 94		if (lowest_vaddr > phdr->p_vaddr)
 95			lowest_vaddr = phdr->p_vaddr;
 96	}
 97
 98	kbuf.image = image;
 99	kbuf.buf_min = lowest_paddr;
100	kbuf.buf_max = ULONG_MAX;
101	kbuf.buf_align = PAGE_SIZE;
102	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
103	kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
104	kbuf.top_down = false;
105	ret = arch_kexec_locate_mem_hole(&kbuf);
106	if (!ret) {
107		*old_pbase = lowest_paddr;
108		*new_pbase = kbuf.mem;
109		image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
110	}
111	return ret;
112}
113
114static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
115{
116	unsigned int *nr_ranges = arg;
117
118	(*nr_ranges)++;
119	return 0;
120}
121
122static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
123{
124	struct crash_mem *cmem = arg;
125
126	cmem->ranges[cmem->nr_ranges].start = res->start;
127	cmem->ranges[cmem->nr_ranges].end = res->end;
128	cmem->nr_ranges++;
129
130	return 0;
131}
132
133static int prepare_elf_headers(void **addr, unsigned long *sz)
134{
135	struct crash_mem *cmem;
136	unsigned int nr_ranges;
137	int ret;
138
139	nr_ranges = 1; /* For exclusion of crashkernel region */
140	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
141
142	cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
143	if (!cmem)
144		return -ENOMEM;
145
146	cmem->max_nr_ranges = nr_ranges;
147	cmem->nr_ranges = 0;
148	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
149	if (ret)
150		goto out;
151
152	/* Exclude crashkernel region */
153	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
154	if (!ret)
155		ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
156
157out:
158	kfree(cmem);
159	return ret;
160}
161
162static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
163				 unsigned long cmdline_len)
164{
165	int elfcorehdr_strlen;
166	char *cmdline_ptr;
167
168	cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
169	if (!cmdline_ptr)
170		return NULL;
171
172	elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
173		image->elf_load_addr);
174
175	if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
176		pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
177		kfree(cmdline_ptr);
178		return NULL;
179	}
180
181	memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
182	/* Ensure it's nul terminated */
183	cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
184	return cmdline_ptr;
185}
186
187static void *elf_kexec_load(struct kimage *image, char *kernel_buf,
188			    unsigned long kernel_len, char *initrd,
189			    unsigned long initrd_len, char *cmdline,
190			    unsigned long cmdline_len)
191{
192	int ret;
193	unsigned long old_kernel_pbase = ULONG_MAX;
194	unsigned long new_kernel_pbase = 0UL;
195	unsigned long initrd_pbase = 0UL;
196	unsigned long headers_sz;
197	unsigned long kernel_start;
198	void *fdt, *headers;
199	struct elfhdr ehdr;
200	struct kexec_buf kbuf;
201	struct kexec_elf_info elf_info;
202	char *modified_cmdline = NULL;
203
204	ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
205	if (ret)
206		return ERR_PTR(ret);
207
208	ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
209			     &old_kernel_pbase, &new_kernel_pbase);
210	if (ret)
211		goto out;
212	kernel_start = image->start;
213	pr_notice("The entry point of kernel at 0x%lx\n", image->start);
214
215	/* Add the kernel binary to the image */
216	ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
217				   old_kernel_pbase, new_kernel_pbase);
218	if (ret)
219		goto out;
220
221	kbuf.image = image;
222	kbuf.buf_min = new_kernel_pbase + kernel_len;
223	kbuf.buf_max = ULONG_MAX;
224
225	/* Add elfcorehdr */
226	if (image->type == KEXEC_TYPE_CRASH) {
227		ret = prepare_elf_headers(&headers, &headers_sz);
228		if (ret) {
229			pr_err("Preparing elf core header failed\n");
230			goto out;
231		}
232
233		kbuf.buffer = headers;
234		kbuf.bufsz = headers_sz;
235		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
236		kbuf.memsz = headers_sz;
237		kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
238		kbuf.top_down = true;
239
240		ret = kexec_add_buffer(&kbuf);
241		if (ret) {
242			vfree(headers);
243			goto out;
244		}
245		image->elf_headers = headers;
246		image->elf_load_addr = kbuf.mem;
247		image->elf_headers_sz = headers_sz;
248
249		pr_debug("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
250			 image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
251
252		/* Setup cmdline for kdump kernel case */
253		modified_cmdline = setup_kdump_cmdline(image, cmdline,
254						       cmdline_len);
255		if (!modified_cmdline) {
256			pr_err("Setting up cmdline for kdump kernel failed\n");
257			ret = -EINVAL;
258			goto out;
259		}
260		cmdline = modified_cmdline;
261	}
262
263#ifdef CONFIG_ARCH_HAS_KEXEC_PURGATORY
264	/* Add purgatory to the image */
265	kbuf.top_down = true;
266	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
267	ret = kexec_load_purgatory(image, &kbuf);
268	if (ret) {
269		pr_err("Error loading purgatory ret=%d\n", ret);
270		goto out;
271	}
272	ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
273					     &kernel_start,
274					     sizeof(kernel_start), 0);
275	if (ret)
276		pr_err("Error update purgatory ret=%d\n", ret);
277#endif /* CONFIG_ARCH_HAS_KEXEC_PURGATORY */
278
279	/* Add the initrd to the image */
280	if (initrd != NULL) {
281		kbuf.buffer = initrd;
282		kbuf.bufsz = kbuf.memsz = initrd_len;
283		kbuf.buf_align = PAGE_SIZE;
284		kbuf.top_down = false;
285		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
286		ret = kexec_add_buffer(&kbuf);
287		if (ret)
288			goto out;
289		initrd_pbase = kbuf.mem;
290		pr_notice("Loaded initrd at 0x%lx\n", initrd_pbase);
291	}
292
293	/* Add the DTB to the image */
294	fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
295					   initrd_len, cmdline, 0);
296	if (!fdt) {
297		pr_err("Error setting up the new device tree.\n");
298		ret = -EINVAL;
299		goto out;
300	}
301
302	fdt_pack(fdt);
303	kbuf.buffer = fdt;
304	kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
305	kbuf.buf_align = PAGE_SIZE;
306	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
307	kbuf.top_down = true;
308	ret = kexec_add_buffer(&kbuf);
309	if (ret) {
310		pr_err("Error add DTB kbuf ret=%d\n", ret);
311		goto out_free_fdt;
312	}
313	/* Cache the fdt buffer address for memory cleanup */
314	image->arch.fdt = fdt;
315	pr_notice("Loaded device tree at 0x%lx\n", kbuf.mem);
316	goto out;
317
318out_free_fdt:
319	kvfree(fdt);
320out:
321	kfree(modified_cmdline);
322	kexec_free_elf_info(&elf_info);
323	return ret ? ERR_PTR(ret) : NULL;
324}
325
326#define RV_X(x, s, n)  (((x) >> (s)) & ((1 << (n)) - 1))
327#define RISCV_IMM_BITS 12
328#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
329#define RISCV_CONST_HIGH_PART(x) \
330	(((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
331#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
332
333#define ENCODE_ITYPE_IMM(x) \
334	(RV_X(x, 0, 12) << 20)
335#define ENCODE_BTYPE_IMM(x) \
336	((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
337	(RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
338#define ENCODE_UTYPE_IMM(x) \
339	(RV_X(x, 12, 20) << 12)
340#define ENCODE_JTYPE_IMM(x) \
341	((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
342	(RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
343#define ENCODE_CBTYPE_IMM(x) \
344	((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
345	(RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
346#define ENCODE_CJTYPE_IMM(x) \
347	((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
348	(RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
349	(RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
350#define ENCODE_UJTYPE_IMM(x) \
351	(ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
352	(ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
353#define ENCODE_UITYPE_IMM(x) \
354	(ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
355
356#define CLEAN_IMM(type, x) \
357	((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
358
359int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
360				     Elf_Shdr *section,
361				     const Elf_Shdr *relsec,
362				     const Elf_Shdr *symtab)
363{
364	const char *strtab, *name, *shstrtab;
365	const Elf_Shdr *sechdrs;
366	Elf64_Rela *relas;
367	int i, r_type;
368
369	/* String & section header string table */
370	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
371	strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
372	shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
373
374	relas = (void *)pi->ehdr + relsec->sh_offset;
375
376	for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
377		const Elf_Sym *sym;	/* symbol to relocate */
378		unsigned long addr;	/* final location after relocation */
379		unsigned long val;	/* relocated symbol value */
380		unsigned long sec_base;	/* relocated symbol value */
381		void *loc;		/* tmp location to modify */
382
383		sym = (void *)pi->ehdr + symtab->sh_offset;
384		sym += ELF64_R_SYM(relas[i].r_info);
385
386		if (sym->st_name)
387			name = strtab + sym->st_name;
388		else
389			name = shstrtab + sechdrs[sym->st_shndx].sh_name;
390
391		loc = pi->purgatory_buf;
392		loc += section->sh_offset;
393		loc += relas[i].r_offset;
394
395		if (sym->st_shndx == SHN_ABS)
396			sec_base = 0;
397		else if (sym->st_shndx >= pi->ehdr->e_shnum) {
398			pr_err("Invalid section %d for symbol %s\n",
399			       sym->st_shndx, name);
400			return -ENOEXEC;
401		} else
402			sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
403
404		val = sym->st_value;
405		val += sec_base;
406		val += relas[i].r_addend;
407
408		addr = section->sh_addr + relas[i].r_offset;
409
410		r_type = ELF64_R_TYPE(relas[i].r_info);
411
412		switch (r_type) {
413		case R_RISCV_BRANCH:
414			*(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
415				 ENCODE_BTYPE_IMM(val - addr);
416			break;
417		case R_RISCV_JAL:
418			*(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
419				 ENCODE_JTYPE_IMM(val - addr);
420			break;
421		/*
422		 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
423		 * sym is expected to be next to R_RISCV_PCREL_HI20
424		 * in purgatory relsec. Handle it like R_RISCV_CALL
425		 * sym, instead of searching the whole relsec.
426		 */
427		case R_RISCV_PCREL_HI20:
428		case R_RISCV_CALL:
429			*(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
430				 ENCODE_UJTYPE_IMM(val - addr);
431			break;
432		case R_RISCV_RVC_BRANCH:
433			*(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
434				 ENCODE_CBTYPE_IMM(val - addr);
435			break;
436		case R_RISCV_RVC_JUMP:
437			*(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
438				 ENCODE_CJTYPE_IMM(val - addr);
439			break;
440		case R_RISCV_ADD32:
441			*(u32 *)loc += val;
442			break;
443		case R_RISCV_SUB32:
444			*(u32 *)loc -= val;
445			break;
446		/* It has been applied by R_RISCV_PCREL_HI20 sym */
447		case R_RISCV_PCREL_LO12_I:
448		case R_RISCV_ALIGN:
449		case R_RISCV_RELAX:
450			break;
451		default:
452			pr_err("Unknown rela relocation: %d\n", r_type);
453			return -ENOEXEC;
454		}
455	}
456	return 0;
457}
458
459const struct kexec_file_ops elf_kexec_ops = {
460	.probe = kexec_elf_probe,
461	.load  = elf_kexec_load,
462};