Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Author: Hanlu Li <lihanlu@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
5 *
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
7 */
8
9#define pr_fmt(fmt) "kmod: " fmt
10
11#include <linux/moduleloader.h>
12#include <linux/elf.h>
13#include <linux/mm.h>
14#include <linux/numa.h>
15#include <linux/vmalloc.h>
16#include <linux/slab.h>
17#include <linux/fs.h>
18#include <linux/ftrace.h>
19#include <linux/string.h>
20#include <linux/kernel.h>
21#include <asm/alternative.h>
22#include <asm/inst.h>
23
24static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
25{
26 if (*rela_stack_top >= RELA_STACK_DEPTH)
27 return -ENOEXEC;
28
29 rela_stack[(*rela_stack_top)++] = stack_value;
30 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
31
32 return 0;
33}
34
35static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
36{
37 if (*rela_stack_top == 0)
38 return -ENOEXEC;
39
40 *stack_value = rela_stack[--(*rela_stack_top)];
41 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
42
43 return 0;
44}
45
46static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
47 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
48{
49 return 0;
50}
51
52static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
53 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
54{
55 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
56 return -EINVAL;
57}
58
59static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
60 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
61{
62 *location = v;
63 return 0;
64}
65
66static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
67 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
68{
69 *(Elf_Addr *)location = v;
70 return 0;
71}
72
73static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
74 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
75{
76 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
77}
78
79static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
80 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
81{
82 return rela_stack_push(v, rela_stack, rela_stack_top);
83}
84
85static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
86 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
87{
88 int err = 0;
89 s64 opr1;
90
91 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
92 if (err)
93 return err;
94 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
95 if (err)
96 return err;
97 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
98 if (err)
99 return err;
100
101 return 0;
102}
103
104static int apply_r_larch_sop_push_plt_pcrel(struct module *mod,
105 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
106 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
107{
108 ptrdiff_t offset = (void *)v - (void *)location;
109
110 if (offset >= SZ_128M)
111 v = module_emit_plt_entry(mod, sechdrs, v);
112
113 if (offset < -SZ_128M)
114 v = module_emit_plt_entry(mod, sechdrs, v);
115
116 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
117}
118
119static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
120 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
121{
122 int err = 0;
123 s64 opr1, opr2, opr3;
124
125 if (type == R_LARCH_SOP_IF_ELSE) {
126 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
127 if (err)
128 return err;
129 }
130
131 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
132 if (err)
133 return err;
134 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
135 if (err)
136 return err;
137
138 switch (type) {
139 case R_LARCH_SOP_AND:
140 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
141 break;
142 case R_LARCH_SOP_ADD:
143 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
144 break;
145 case R_LARCH_SOP_SUB:
146 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
147 break;
148 case R_LARCH_SOP_SL:
149 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
150 break;
151 case R_LARCH_SOP_SR:
152 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
153 break;
154 case R_LARCH_SOP_IF_ELSE:
155 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
156 break;
157 default:
158 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
159 return -EINVAL;
160 }
161
162 return err;
163}
164
165static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
166 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
167{
168 int err = 0;
169 s64 opr1;
170 union loongarch_instruction *insn = (union loongarch_instruction *)location;
171
172 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
173 if (err)
174 return err;
175
176 switch (type) {
177 case R_LARCH_SOP_POP_32_U_10_12:
178 if (!unsigned_imm_check(opr1, 12))
179 goto overflow;
180
181 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
182 insn->reg2i12_format.immediate = opr1 & 0xfff;
183 return 0;
184 case R_LARCH_SOP_POP_32_S_10_12:
185 if (!signed_imm_check(opr1, 12))
186 goto overflow;
187
188 insn->reg2i12_format.immediate = opr1 & 0xfff;
189 return 0;
190 case R_LARCH_SOP_POP_32_S_10_16:
191 if (!signed_imm_check(opr1, 16))
192 goto overflow;
193
194 insn->reg2i16_format.immediate = opr1 & 0xffff;
195 return 0;
196 case R_LARCH_SOP_POP_32_S_10_16_S2:
197 if (opr1 % 4)
198 goto unaligned;
199
200 if (!signed_imm_check(opr1, 18))
201 goto overflow;
202
203 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
204 return 0;
205 case R_LARCH_SOP_POP_32_S_5_20:
206 if (!signed_imm_check(opr1, 20))
207 goto overflow;
208
209 insn->reg1i20_format.immediate = (opr1) & 0xfffff;
210 return 0;
211 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
212 if (opr1 % 4)
213 goto unaligned;
214
215 if (!signed_imm_check(opr1, 23))
216 goto overflow;
217
218 opr1 >>= 2;
219 insn->reg1i21_format.immediate_l = opr1 & 0xffff;
220 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
221 return 0;
222 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
223 if (opr1 % 4)
224 goto unaligned;
225
226 if (!signed_imm_check(opr1, 28))
227 goto overflow;
228
229 opr1 >>= 2;
230 insn->reg0i26_format.immediate_l = opr1 & 0xffff;
231 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
232 return 0;
233 case R_LARCH_SOP_POP_32_U:
234 if (!unsigned_imm_check(opr1, 32))
235 goto overflow;
236
237 /* (*(uint32_t *) PC) = opr */
238 *location = (u32)opr1;
239 return 0;
240 default:
241 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
242 return -EINVAL;
243 }
244
245overflow:
246 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
247 mod->name, opr1, __func__, type);
248 return -ENOEXEC;
249
250unaligned:
251 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
252 mod->name, opr1, __func__, type);
253 return -ENOEXEC;
254}
255
256static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
257 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
258{
259 switch (type) {
260 case R_LARCH_ADD32:
261 *(s32 *)location += v;
262 return 0;
263 case R_LARCH_ADD64:
264 *(s64 *)location += v;
265 return 0;
266 case R_LARCH_SUB32:
267 *(s32 *)location -= v;
268 return 0;
269 case R_LARCH_SUB64:
270 *(s64 *)location -= v;
271 return 0;
272 default:
273 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
274 return -EINVAL;
275 }
276}
277
278static int apply_r_larch_b26(struct module *mod,
279 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
280 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
281{
282 ptrdiff_t offset = (void *)v - (void *)location;
283 union loongarch_instruction *insn = (union loongarch_instruction *)location;
284
285 if (offset >= SZ_128M)
286 v = module_emit_plt_entry(mod, sechdrs, v);
287
288 if (offset < -SZ_128M)
289 v = module_emit_plt_entry(mod, sechdrs, v);
290
291 offset = (void *)v - (void *)location;
292
293 if (offset & 3) {
294 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
295 mod->name, (long long)offset, type);
296 return -ENOEXEC;
297 }
298
299 if (!signed_imm_check(offset, 28)) {
300 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
301 mod->name, (long long)offset, type);
302 return -ENOEXEC;
303 }
304
305 offset >>= 2;
306 insn->reg0i26_format.immediate_l = offset & 0xffff;
307 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
308
309 return 0;
310}
311
312static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
313 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
314{
315 union loongarch_instruction *insn = (union loongarch_instruction *)location;
316 /* Use s32 for a sign-extension deliberately. */
317 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
318 (void *)((Elf_Addr)location & ~0xfff);
319 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
320 ptrdiff_t offset_rem = (void *)v - (void *)anchor;
321
322 switch (type) {
323 case R_LARCH_PCALA_LO12:
324 insn->reg2i12_format.immediate = v & 0xfff;
325 break;
326 case R_LARCH_PCALA_HI20:
327 v = offset_hi20 >> 12;
328 insn->reg1i20_format.immediate = v & 0xfffff;
329 break;
330 case R_LARCH_PCALA64_LO20:
331 v = offset_rem >> 32;
332 insn->reg1i20_format.immediate = v & 0xfffff;
333 break;
334 case R_LARCH_PCALA64_HI12:
335 v = offset_rem >> 52;
336 insn->reg2i12_format.immediate = v & 0xfff;
337 break;
338 default:
339 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
340 return -EINVAL;
341 }
342
343 return 0;
344}
345
346static int apply_r_larch_got_pc(struct module *mod,
347 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
348 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
349{
350 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v);
351
352 if (!got)
353 return -EINVAL;
354
355 switch (type) {
356 case R_LARCH_GOT_PC_LO12:
357 type = R_LARCH_PCALA_LO12;
358 break;
359 case R_LARCH_GOT_PC_HI20:
360 type = R_LARCH_PCALA_HI20;
361 break;
362 default:
363 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
364 return -EINVAL;
365 }
366
367 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
368}
369
370static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v,
371 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
372{
373 ptrdiff_t offset = (void *)v - (void *)location;
374
375 *(u32 *)location = offset;
376 return 0;
377}
378
379static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v,
380 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
381{
382 ptrdiff_t offset = (void *)v - (void *)location;
383
384 *(u64 *)location = offset;
385 return 0;
386}
387
388/*
389 * reloc_handlers_rela() - Apply a particular relocation to a module
390 * @mod: the module to apply the reloc to
391 * @location: the address at which the reloc is to be applied
392 * @v: the value of the reloc, with addend for RELA-style
393 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
394 * @rela_stac_top: where the stack operation(pop/push) applies to
395 *
396 * Return: 0 upon success, else -ERRNO
397 */
398typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
399 s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
400
401/* The handlers for known reloc types */
402static reloc_rela_handler reloc_rela_handlers[] = {
403 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error,
404
405 [R_LARCH_NONE] = apply_r_larch_none,
406 [R_LARCH_32] = apply_r_larch_32,
407 [R_LARCH_64] = apply_r_larch_64,
408 [R_LARCH_MARK_LA] = apply_r_larch_none,
409 [R_LARCH_MARK_PCREL] = apply_r_larch_none,
410 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel,
411 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute,
412 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup,
413 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
414 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
415 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
416 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
417 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel,
418 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel,
419};
420
421int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
422 unsigned int symindex, unsigned int relsec,
423 struct module *mod)
424{
425 int i, err;
426 unsigned int type;
427 s64 rela_stack[RELA_STACK_DEPTH];
428 size_t rela_stack_top = 0;
429 reloc_rela_handler handler;
430 void *location;
431 Elf_Addr v;
432 Elf_Sym *sym;
433 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
434
435 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
436 sechdrs[relsec].sh_info);
437
438 rela_stack_top = 0;
439 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
440 /* This is where to make the change */
441 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
442 /* This is the symbol it is referring to */
443 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
444 if (IS_ERR_VALUE(sym->st_value)) {
445 /* Ignore unresolved weak symbol */
446 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
447 continue;
448 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
449 return -ENOENT;
450 }
451
452 type = ELF_R_TYPE(rel[i].r_info);
453
454 if (type < ARRAY_SIZE(reloc_rela_handlers))
455 handler = reloc_rela_handlers[type];
456 else
457 handler = NULL;
458
459 if (!handler) {
460 pr_err("%s: Unknown relocation type %u\n", mod->name, type);
461 return -EINVAL;
462 }
463
464 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
465 (int)ELF_R_TYPE(rel[i].r_info),
466 sym->st_value, rel[i].r_addend, (u64)location);
467
468 v = sym->st_value + rel[i].r_addend;
469 switch (type) {
470 case R_LARCH_B26:
471 err = apply_r_larch_b26(mod, sechdrs, location,
472 v, rela_stack, &rela_stack_top, type);
473 break;
474 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12:
475 err = apply_r_larch_got_pc(mod, sechdrs, location,
476 v, rela_stack, &rela_stack_top, type);
477 break;
478 case R_LARCH_SOP_PUSH_PLT_PCREL:
479 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location,
480 v, rela_stack, &rela_stack_top, type);
481 break;
482 default:
483 err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
484 }
485 if (err)
486 return err;
487 }
488
489 return 0;
490}
491
492void *module_alloc(unsigned long size)
493{
494 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
495 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
496}
497
498static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
499 const Elf_Shdr *sechdrs, struct module *mod)
500{
501#ifdef CONFIG_DYNAMIC_FTRACE
502 struct plt_entry *ftrace_plts;
503
504 ftrace_plts = (void *)sechdrs->sh_addr;
505
506 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
507
508 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
509 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
510
511 mod->arch.ftrace_trampolines = ftrace_plts;
512#endif
513}
514
515int module_finalize(const Elf_Ehdr *hdr,
516 const Elf_Shdr *sechdrs, struct module *mod)
517{
518 const Elf_Shdr *s, *se;
519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
520
521 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
522 if (!strcmp(".altinstructions", secstrs + s->sh_name))
523 apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
524 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
525 module_init_ftrace_plt(hdr, s, mod);
526 }
527
528 return 0;
529}
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Author: Hanlu Li <lihanlu@loongson.cn>
4 * Huacai Chen <chenhuacai@loongson.cn>
5 *
6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
7 */
8
9#define pr_fmt(fmt) "kmod: " fmt
10
11#include <linux/moduleloader.h>
12#include <linux/elf.h>
13#include <linux/mm.h>
14#include <linux/numa.h>
15#include <linux/vmalloc.h>
16#include <linux/slab.h>
17#include <linux/fs.h>
18#include <linux/ftrace.h>
19#include <linux/string.h>
20#include <linux/kernel.h>
21#include <asm/alternative.h>
22#include <asm/inst.h>
23#include <asm/unwind.h>
24
25static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
26{
27 if (*rela_stack_top >= RELA_STACK_DEPTH)
28 return -ENOEXEC;
29
30 rela_stack[(*rela_stack_top)++] = stack_value;
31 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
32
33 return 0;
34}
35
36static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
37{
38 if (*rela_stack_top == 0)
39 return -ENOEXEC;
40
41 *stack_value = rela_stack[--(*rela_stack_top)];
42 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
43
44 return 0;
45}
46
47static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
48 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
49{
50 return 0;
51}
52
53static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
54 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
55{
56 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
57 return -EINVAL;
58}
59
60static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
61 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
62{
63 *location = v;
64 return 0;
65}
66
67static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
68 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
69{
70 *(Elf_Addr *)location = v;
71 return 0;
72}
73
74static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
75 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
76{
77 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
78}
79
80static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
81 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
82{
83 return rela_stack_push(v, rela_stack, rela_stack_top);
84}
85
86static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
87 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
88{
89 int err = 0;
90 s64 opr1;
91
92 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
93 if (err)
94 return err;
95 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
96 if (err)
97 return err;
98 err = rela_stack_push(opr1, rela_stack, rela_stack_top);
99 if (err)
100 return err;
101
102 return 0;
103}
104
105static int apply_r_larch_sop_push_plt_pcrel(struct module *mod,
106 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
107 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
108{
109 ptrdiff_t offset = (void *)v - (void *)location;
110
111 if (offset >= SZ_128M)
112 v = module_emit_plt_entry(mod, sechdrs, v);
113
114 if (offset < -SZ_128M)
115 v = module_emit_plt_entry(mod, sechdrs, v);
116
117 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
118}
119
120static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
121 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
122{
123 int err = 0;
124 s64 opr1, opr2, opr3;
125
126 if (type == R_LARCH_SOP_IF_ELSE) {
127 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
128 if (err)
129 return err;
130 }
131
132 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
133 if (err)
134 return err;
135 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
136 if (err)
137 return err;
138
139 switch (type) {
140 case R_LARCH_SOP_AND:
141 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
142 break;
143 case R_LARCH_SOP_ADD:
144 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
145 break;
146 case R_LARCH_SOP_SUB:
147 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
148 break;
149 case R_LARCH_SOP_SL:
150 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
151 break;
152 case R_LARCH_SOP_SR:
153 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
154 break;
155 case R_LARCH_SOP_IF_ELSE:
156 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
157 break;
158 default:
159 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
160 return -EINVAL;
161 }
162
163 return err;
164}
165
166static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
167 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
168{
169 int err = 0;
170 s64 opr1;
171 union loongarch_instruction *insn = (union loongarch_instruction *)location;
172
173 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
174 if (err)
175 return err;
176
177 switch (type) {
178 case R_LARCH_SOP_POP_32_U_10_12:
179 if (!unsigned_imm_check(opr1, 12))
180 goto overflow;
181
182 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
183 insn->reg2i12_format.immediate = opr1 & 0xfff;
184 return 0;
185 case R_LARCH_SOP_POP_32_S_10_12:
186 if (!signed_imm_check(opr1, 12))
187 goto overflow;
188
189 insn->reg2i12_format.immediate = opr1 & 0xfff;
190 return 0;
191 case R_LARCH_SOP_POP_32_S_10_16:
192 if (!signed_imm_check(opr1, 16))
193 goto overflow;
194
195 insn->reg2i16_format.immediate = opr1 & 0xffff;
196 return 0;
197 case R_LARCH_SOP_POP_32_S_10_16_S2:
198 if (opr1 % 4)
199 goto unaligned;
200
201 if (!signed_imm_check(opr1, 18))
202 goto overflow;
203
204 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
205 return 0;
206 case R_LARCH_SOP_POP_32_S_5_20:
207 if (!signed_imm_check(opr1, 20))
208 goto overflow;
209
210 insn->reg1i20_format.immediate = (opr1) & 0xfffff;
211 return 0;
212 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
213 if (opr1 % 4)
214 goto unaligned;
215
216 if (!signed_imm_check(opr1, 23))
217 goto overflow;
218
219 opr1 >>= 2;
220 insn->reg1i21_format.immediate_l = opr1 & 0xffff;
221 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
222 return 0;
223 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
224 if (opr1 % 4)
225 goto unaligned;
226
227 if (!signed_imm_check(opr1, 28))
228 goto overflow;
229
230 opr1 >>= 2;
231 insn->reg0i26_format.immediate_l = opr1 & 0xffff;
232 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
233 return 0;
234 case R_LARCH_SOP_POP_32_U:
235 if (!unsigned_imm_check(opr1, 32))
236 goto overflow;
237
238 /* (*(uint32_t *) PC) = opr */
239 *location = (u32)opr1;
240 return 0;
241 default:
242 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
243 return -EINVAL;
244 }
245
246overflow:
247 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
248 mod->name, opr1, __func__, type);
249 return -ENOEXEC;
250
251unaligned:
252 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
253 mod->name, opr1, __func__, type);
254 return -ENOEXEC;
255}
256
257static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
258 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
259{
260 switch (type) {
261 case R_LARCH_ADD32:
262 *(s32 *)location += v;
263 return 0;
264 case R_LARCH_ADD64:
265 *(s64 *)location += v;
266 return 0;
267 case R_LARCH_SUB32:
268 *(s32 *)location -= v;
269 return 0;
270 case R_LARCH_SUB64:
271 *(s64 *)location -= v;
272 return 0;
273 default:
274 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
275 return -EINVAL;
276 }
277}
278
279static int apply_r_larch_b26(struct module *mod,
280 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
281 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
282{
283 ptrdiff_t offset = (void *)v - (void *)location;
284 union loongarch_instruction *insn = (union loongarch_instruction *)location;
285
286 if (offset >= SZ_128M)
287 v = module_emit_plt_entry(mod, sechdrs, v);
288
289 if (offset < -SZ_128M)
290 v = module_emit_plt_entry(mod, sechdrs, v);
291
292 offset = (void *)v - (void *)location;
293
294 if (offset & 3) {
295 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
296 mod->name, (long long)offset, type);
297 return -ENOEXEC;
298 }
299
300 if (!signed_imm_check(offset, 28)) {
301 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
302 mod->name, (long long)offset, type);
303 return -ENOEXEC;
304 }
305
306 offset >>= 2;
307 insn->reg0i26_format.immediate_l = offset & 0xffff;
308 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
309
310 return 0;
311}
312
313static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
314 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
315{
316 union loongarch_instruction *insn = (union loongarch_instruction *)location;
317 /* Use s32 for a sign-extension deliberately. */
318 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
319 (void *)((Elf_Addr)location & ~0xfff);
320 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
321 ptrdiff_t offset_rem = (void *)v - (void *)anchor;
322
323 switch (type) {
324 case R_LARCH_PCALA_LO12:
325 insn->reg2i12_format.immediate = v & 0xfff;
326 break;
327 case R_LARCH_PCALA_HI20:
328 v = offset_hi20 >> 12;
329 insn->reg1i20_format.immediate = v & 0xfffff;
330 break;
331 case R_LARCH_PCALA64_LO20:
332 v = offset_rem >> 32;
333 insn->reg1i20_format.immediate = v & 0xfffff;
334 break;
335 case R_LARCH_PCALA64_HI12:
336 v = offset_rem >> 52;
337 insn->reg2i12_format.immediate = v & 0xfff;
338 break;
339 default:
340 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
341 return -EINVAL;
342 }
343
344 return 0;
345}
346
347static int apply_r_larch_got_pc(struct module *mod,
348 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
349 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
350{
351 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v);
352
353 if (!got)
354 return -EINVAL;
355
356 switch (type) {
357 case R_LARCH_GOT_PC_LO12:
358 type = R_LARCH_PCALA_LO12;
359 break;
360 case R_LARCH_GOT_PC_HI20:
361 type = R_LARCH_PCALA_HI20;
362 break;
363 default:
364 pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
365 return -EINVAL;
366 }
367
368 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
369}
370
371static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v,
372 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
373{
374 ptrdiff_t offset = (void *)v - (void *)location;
375
376 *(u32 *)location = offset;
377 return 0;
378}
379
380static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v,
381 s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
382{
383 ptrdiff_t offset = (void *)v - (void *)location;
384
385 *(u64 *)location = offset;
386 return 0;
387}
388
389/*
390 * reloc_handlers_rela() - Apply a particular relocation to a module
391 * @mod: the module to apply the reloc to
392 * @location: the address at which the reloc is to be applied
393 * @v: the value of the reloc, with addend for RELA-style
394 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
395 * @rela_stac_top: where the stack operation(pop/push) applies to
396 *
397 * Return: 0 upon success, else -ERRNO
398 */
399typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
400 s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
401
402/* The handlers for known reloc types */
403static reloc_rela_handler reloc_rela_handlers[] = {
404 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error,
405
406 [R_LARCH_NONE] = apply_r_larch_none,
407 [R_LARCH_32] = apply_r_larch_32,
408 [R_LARCH_64] = apply_r_larch_64,
409 [R_LARCH_MARK_LA] = apply_r_larch_none,
410 [R_LARCH_MARK_PCREL] = apply_r_larch_none,
411 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel,
412 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute,
413 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup,
414 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
415 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
416 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
417 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
418 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel,
419 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel,
420};
421
422int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
423 unsigned int symindex, unsigned int relsec,
424 struct module *mod)
425{
426 int i, err;
427 unsigned int type;
428 s64 rela_stack[RELA_STACK_DEPTH];
429 size_t rela_stack_top = 0;
430 reloc_rela_handler handler;
431 void *location;
432 Elf_Addr v;
433 Elf_Sym *sym;
434 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
435
436 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
437 sechdrs[relsec].sh_info);
438
439 rela_stack_top = 0;
440 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
441 /* This is where to make the change */
442 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
443 /* This is the symbol it is referring to */
444 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
445 if (IS_ERR_VALUE(sym->st_value)) {
446 /* Ignore unresolved weak symbol */
447 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
448 continue;
449 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
450 return -ENOENT;
451 }
452
453 type = ELF_R_TYPE(rel[i].r_info);
454
455 if (type < ARRAY_SIZE(reloc_rela_handlers))
456 handler = reloc_rela_handlers[type];
457 else
458 handler = NULL;
459
460 if (!handler) {
461 pr_err("%s: Unknown relocation type %u\n", mod->name, type);
462 return -EINVAL;
463 }
464
465 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
466 (int)ELF_R_TYPE(rel[i].r_info),
467 sym->st_value, rel[i].r_addend, (u64)location);
468
469 v = sym->st_value + rel[i].r_addend;
470 switch (type) {
471 case R_LARCH_B26:
472 err = apply_r_larch_b26(mod, sechdrs, location,
473 v, rela_stack, &rela_stack_top, type);
474 break;
475 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12:
476 err = apply_r_larch_got_pc(mod, sechdrs, location,
477 v, rela_stack, &rela_stack_top, type);
478 break;
479 case R_LARCH_SOP_PUSH_PLT_PCREL:
480 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location,
481 v, rela_stack, &rela_stack_top, type);
482 break;
483 default:
484 err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
485 }
486 if (err)
487 return err;
488 }
489
490 return 0;
491}
492
493void *module_alloc(unsigned long size)
494{
495 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
496 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
497}
498
499static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
500 const Elf_Shdr *sechdrs, struct module *mod)
501{
502#ifdef CONFIG_DYNAMIC_FTRACE
503 struct plt_entry *ftrace_plts;
504
505 ftrace_plts = (void *)sechdrs->sh_addr;
506
507 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
508
509 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
510 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
511
512 mod->arch.ftrace_trampolines = ftrace_plts;
513#endif
514}
515
516int module_finalize(const Elf_Ehdr *hdr,
517 const Elf_Shdr *sechdrs, struct module *mod)
518{
519 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
520 const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL;
521
522 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
523 if (!strcmp(".altinstructions", secstrs + s->sh_name))
524 alt = s;
525 if (!strcmp(".orc_unwind", secstrs + s->sh_name))
526 orc = s;
527 if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name))
528 orc_ip = s;
529 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
530 ftrace = s;
531 }
532
533 if (alt)
534 apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size);
535
536 if (orc && orc_ip)
537 unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size);
538
539 if (ftrace)
540 module_init_ftrace_plt(hdr, ftrace, mod);
541
542 return 0;
543}