Loading...
1/* Kernel module help for PPC64.
2 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17*/
18
19#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20
21#include <linux/module.h>
22#include <linux/elf.h>
23#include <linux/moduleloader.h>
24#include <linux/err.h>
25#include <linux/vmalloc.h>
26#include <linux/ftrace.h>
27#include <linux/bug.h>
28#include <linux/uaccess.h>
29#include <asm/module.h>
30#include <asm/firmware.h>
31#include <asm/code-patching.h>
32#include <linux/sort.h>
33#include <asm/setup.h>
34#include <asm/sections.h>
35
36/* FIXME: We don't do .init separately. To do this, we'd need to have
37 a separate r2 value in the init and core section, and stub between
38 them, too.
39
40 Using a magic allocator which places modules within 32MB solves
41 this, and makes other things simpler. Anton?
42 --RR. */
43
44#ifdef PPC64_ELF_ABI_v2
45
46/* An address is simply the address of the function. */
47typedef unsigned long func_desc_t;
48
49static func_desc_t func_desc(unsigned long addr)
50{
51 return addr;
52}
53static unsigned long func_addr(unsigned long addr)
54{
55 return addr;
56}
57static unsigned long stub_func_addr(func_desc_t func)
58{
59 return func;
60}
61
62/* PowerPC64 specific values for the Elf64_Sym st_other field. */
63#define STO_PPC64_LOCAL_BIT 5
64#define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT)
65#define PPC64_LOCAL_ENTRY_OFFSET(other) \
66 (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)
67
68static unsigned int local_entry_offset(const Elf64_Sym *sym)
69{
70 /* sym->st_other indicates offset to local entry point
71 * (otherwise it will assume r12 is the address of the start
72 * of function and try to derive r2 from it). */
73 return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other);
74}
75#else
76
77/* An address is address of the OPD entry, which contains address of fn. */
78typedef struct ppc64_opd_entry func_desc_t;
79
80static func_desc_t func_desc(unsigned long addr)
81{
82 return *(struct ppc64_opd_entry *)addr;
83}
84static unsigned long func_addr(unsigned long addr)
85{
86 return func_desc(addr).funcaddr;
87}
88static unsigned long stub_func_addr(func_desc_t func)
89{
90 return func.funcaddr;
91}
92static unsigned int local_entry_offset(const Elf64_Sym *sym)
93{
94 return 0;
95}
96
97void *dereference_module_function_descriptor(struct module *mod, void *ptr)
98{
99 if (ptr < (void *)mod->arch.start_opd ||
100 ptr >= (void *)mod->arch.end_opd)
101 return ptr;
102
103 return dereference_function_descriptor(ptr);
104}
105#endif
106
107#define STUB_MAGIC 0x73747562 /* stub */
108
109/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
110 the kernel itself). But on PPC64, these need to be used for every
111 jump, actually, to reset r2 (TOC+0x8000). */
112struct ppc64_stub_entry
113{
114 /* 28 byte jump instruction sequence (7 instructions). We only
115 * need 6 instructions on ABIv2 but we always allocate 7 so
116 * so we don't have to modify the trampoline load instruction. */
117 u32 jump[7];
118 /* Used by ftrace to identify stubs */
119 u32 magic;
120 /* Data for the above code */
121 func_desc_t funcdata;
122};
123
124/*
125 * PPC64 uses 24 bit jumps, but we need to jump into other modules or
126 * the kernel which may be further. So we jump to a stub.
127 *
128 * For ELFv1 we need to use this to set up the new r2 value (aka TOC
129 * pointer). For ELFv2 it's the callee's responsibility to set up the
130 * new r2, but for both we need to save the old r2.
131 *
132 * We could simply patch the new r2 value and function pointer into
133 * the stub, but it's significantly shorter to put these values at the
134 * end of the stub code, and patch the stub address (32-bits relative
135 * to the TOC ptr, r2) into the stub.
136 */
137
138static u32 ppc64_stub_insns[] = {
139 0x3d620000, /* addis r11,r2, <high> */
140 0x396b0000, /* addi r11,r11, <low> */
141 /* Save current r2 value in magic place on the stack. */
142 0xf8410000|R2_STACK_OFFSET, /* std r2,R2_STACK_OFFSET(r1) */
143 0xe98b0020, /* ld r12,32(r11) */
144#ifdef PPC64_ELF_ABI_v1
145 /* Set up new r2 from function descriptor */
146 0xe84b0028, /* ld r2,40(r11) */
147#endif
148 0x7d8903a6, /* mtctr r12 */
149 0x4e800420 /* bctr */
150};
151
152#ifdef CONFIG_DYNAMIC_FTRACE
153int module_trampoline_target(struct module *mod, unsigned long addr,
154 unsigned long *target)
155{
156 struct ppc64_stub_entry *stub;
157 func_desc_t funcdata;
158 u32 magic;
159
160 if (!within_module_core(addr, mod)) {
161 pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
162 return -EFAULT;
163 }
164
165 stub = (struct ppc64_stub_entry *)addr;
166
167 if (probe_kernel_read(&magic, &stub->magic, sizeof(magic))) {
168 pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
169 return -EFAULT;
170 }
171
172 if (magic != STUB_MAGIC) {
173 pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
174 return -EFAULT;
175 }
176
177 if (probe_kernel_read(&funcdata, &stub->funcdata, sizeof(funcdata))) {
178 pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
179 return -EFAULT;
180 }
181
182 *target = stub_func_addr(funcdata);
183
184 return 0;
185}
186#endif
187
188/* Count how many different 24-bit relocations (different symbol,
189 different addend) */
190static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num)
191{
192 unsigned int i, r_info, r_addend, _count_relocs;
193
194 /* FIXME: Only count external ones --RR */
195 _count_relocs = 0;
196 r_info = 0;
197 r_addend = 0;
198 for (i = 0; i < num; i++)
199 /* Only count 24-bit relocs, others don't need stubs */
200 if (ELF64_R_TYPE(rela[i].r_info) == R_PPC_REL24 &&
201 (r_info != ELF64_R_SYM(rela[i].r_info) ||
202 r_addend != rela[i].r_addend)) {
203 _count_relocs++;
204 r_info = ELF64_R_SYM(rela[i].r_info);
205 r_addend = rela[i].r_addend;
206 }
207
208 return _count_relocs;
209}
210
211static int relacmp(const void *_x, const void *_y)
212{
213 const Elf64_Rela *x, *y;
214
215 y = (Elf64_Rela *)_x;
216 x = (Elf64_Rela *)_y;
217
218 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
219 * make the comparison cheaper/faster. It won't affect the sorting or
220 * the counting algorithms' performance
221 */
222 if (x->r_info < y->r_info)
223 return -1;
224 else if (x->r_info > y->r_info)
225 return 1;
226 else if (x->r_addend < y->r_addend)
227 return -1;
228 else if (x->r_addend > y->r_addend)
229 return 1;
230 else
231 return 0;
232}
233
234static void relaswap(void *_x, void *_y, int size)
235{
236 uint64_t *x, *y, tmp;
237 int i;
238
239 y = (uint64_t *)_x;
240 x = (uint64_t *)_y;
241
242 for (i = 0; i < sizeof(Elf64_Rela) / sizeof(uint64_t); i++) {
243 tmp = x[i];
244 x[i] = y[i];
245 y[i] = tmp;
246 }
247}
248
249/* Get size of potential trampolines required. */
250static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
251 const Elf64_Shdr *sechdrs)
252{
253 /* One extra reloc so it's always 0-funcaddr terminated */
254 unsigned long relocs = 1;
255 unsigned i;
256
257 /* Every relocated section... */
258 for (i = 1; i < hdr->e_shnum; i++) {
259 if (sechdrs[i].sh_type == SHT_RELA) {
260 pr_debug("Found relocations in section %u\n", i);
261 pr_debug("Ptr: %p. Number: %Lu\n",
262 (void *)sechdrs[i].sh_addr,
263 sechdrs[i].sh_size / sizeof(Elf64_Rela));
264
265 /* Sort the relocation information based on a symbol and
266 * addend key. This is a stable O(n*log n) complexity
267 * alogrithm but it will reduce the complexity of
268 * count_relocs() to linear complexity O(n)
269 */
270 sort((void *)sechdrs[i].sh_addr,
271 sechdrs[i].sh_size / sizeof(Elf64_Rela),
272 sizeof(Elf64_Rela), relacmp, relaswap);
273
274 relocs += count_relocs((void *)sechdrs[i].sh_addr,
275 sechdrs[i].sh_size
276 / sizeof(Elf64_Rela));
277 }
278 }
279
280#ifdef CONFIG_DYNAMIC_FTRACE
281 /* make the trampoline to the ftrace_caller */
282 relocs++;
283#endif
284
285 pr_debug("Looks like a total of %lu stubs, max\n", relocs);
286 return relocs * sizeof(struct ppc64_stub_entry);
287}
288
289/* Still needed for ELFv2, for .TOC. */
290static void dedotify_versions(struct modversion_info *vers,
291 unsigned long size)
292{
293 struct modversion_info *end;
294
295 for (end = (void *)vers + size; vers < end; vers++)
296 if (vers->name[0] == '.') {
297 memmove(vers->name, vers->name+1, strlen(vers->name));
298 }
299}
300
301/*
302 * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC.
303 * seem to be defined (value set later).
304 */
305static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
306{
307 unsigned int i;
308
309 for (i = 1; i < numsyms; i++) {
310 if (syms[i].st_shndx == SHN_UNDEF) {
311 char *name = strtab + syms[i].st_name;
312 if (name[0] == '.') {
313 if (strcmp(name+1, "TOC.") == 0)
314 syms[i].st_shndx = SHN_ABS;
315 syms[i].st_name++;
316 }
317 }
318 }
319}
320
321static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs,
322 const char *strtab,
323 unsigned int symindex)
324{
325 unsigned int i, numsyms;
326 Elf64_Sym *syms;
327
328 syms = (Elf64_Sym *)sechdrs[symindex].sh_addr;
329 numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym);
330
331 for (i = 1; i < numsyms; i++) {
332 if (syms[i].st_shndx == SHN_ABS
333 && strcmp(strtab + syms[i].st_name, "TOC.") == 0)
334 return &syms[i];
335 }
336 return NULL;
337}
338
339int module_frob_arch_sections(Elf64_Ehdr *hdr,
340 Elf64_Shdr *sechdrs,
341 char *secstrings,
342 struct module *me)
343{
344 unsigned int i;
345
346 /* Find .toc and .stubs sections, symtab and strtab */
347 for (i = 1; i < hdr->e_shnum; i++) {
348 char *p;
349 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
350 me->arch.stubs_section = i;
351 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) {
352 me->arch.toc_section = i;
353 if (sechdrs[i].sh_addralign < 8)
354 sechdrs[i].sh_addralign = 8;
355 }
356 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
357 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
358 sechdrs[i].sh_size);
359 else if (!strcmp(secstrings + sechdrs[i].sh_name, ".opd")) {
360 me->arch.start_opd = sechdrs[i].sh_addr;
361 me->arch.end_opd = sechdrs[i].sh_addr +
362 sechdrs[i].sh_size;
363 }
364
365 /* We don't handle .init for the moment: rename to _init */
366 while ((p = strstr(secstrings + sechdrs[i].sh_name, ".init")))
367 p[0] = '_';
368
369 if (sechdrs[i].sh_type == SHT_SYMTAB)
370 dedotify((void *)hdr + sechdrs[i].sh_offset,
371 sechdrs[i].sh_size / sizeof(Elf64_Sym),
372 (void *)hdr
373 + sechdrs[sechdrs[i].sh_link].sh_offset);
374 }
375
376 if (!me->arch.stubs_section) {
377 pr_err("%s: doesn't contain .stubs.\n", me->name);
378 return -ENOEXEC;
379 }
380
381 /* If we don't have a .toc, just use .stubs. We need to set r2
382 to some reasonable value in case the module calls out to
383 other functions via a stub, or if a function pointer escapes
384 the module by some means. */
385 if (!me->arch.toc_section)
386 me->arch.toc_section = me->arch.stubs_section;
387
388 /* Override the stubs size */
389 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
390 return 0;
391}
392
393/*
394 * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
395 * value maximum span in an instruction which uses a signed offset). Round down
396 * to a 256 byte boundary for the odd case where we are setting up r2 without a
397 * .toc section.
398 */
399static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me)
400{
401 return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
402}
403
404/* Both low and high 16 bits are added as SIGNED additions, so if low
405 16 bits has high bit set, high 16 bits must be adjusted. These
406 macros do that (stolen from binutils). */
407#define PPC_LO(v) ((v) & 0xffff)
408#define PPC_HI(v) (((v) >> 16) & 0xffff)
409#define PPC_HA(v) PPC_HI ((v) + 0x8000)
410
411/* Patch stub to reference function and correct r2 value. */
412static inline int create_stub(const Elf64_Shdr *sechdrs,
413 struct ppc64_stub_entry *entry,
414 unsigned long addr,
415 struct module *me)
416{
417 long reladdr;
418
419 memcpy(entry->jump, ppc64_stub_insns, sizeof(ppc64_stub_insns));
420
421 /* Stub uses address relative to r2. */
422 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
423 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
424 pr_err("%s: Address %p of stub out of range of %p.\n",
425 me->name, (void *)reladdr, (void *)my_r2);
426 return 0;
427 }
428 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
429
430 entry->jump[0] |= PPC_HA(reladdr);
431 entry->jump[1] |= PPC_LO(reladdr);
432 entry->funcdata = func_desc(addr);
433 entry->magic = STUB_MAGIC;
434
435 return 1;
436}
437
438/* Create stub to jump to function described in this OPD/ptr: we need the
439 stub to set up the TOC ptr (r2) for the function. */
440static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
441 unsigned long addr,
442 struct module *me)
443{
444 struct ppc64_stub_entry *stubs;
445 unsigned int i, num_stubs;
446
447 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
448
449 /* Find this stub, or if that fails, the next avail. entry */
450 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
451 for (i = 0; stub_func_addr(stubs[i].funcdata); i++) {
452 if (WARN_ON(i >= num_stubs))
453 return 0;
454
455 if (stub_func_addr(stubs[i].funcdata) == func_addr(addr))
456 return (unsigned long)&stubs[i];
457 }
458
459 if (!create_stub(sechdrs, &stubs[i], addr, me))
460 return 0;
461
462 return (unsigned long)&stubs[i];
463}
464
465#ifdef CC_USING_MPROFILE_KERNEL
466static bool is_early_mcount_callsite(u32 *instruction)
467{
468 /*
469 * Check if this is one of the -mprofile-kernel sequences.
470 */
471 if (instruction[-1] == PPC_INST_STD_LR &&
472 instruction[-2] == PPC_INST_MFLR)
473 return true;
474
475 if (instruction[-1] == PPC_INST_MFLR)
476 return true;
477
478 return false;
479}
480
481/*
482 * In case of _mcount calls, do not save the current callee's TOC (in r2) into
483 * the original caller's stack frame. If we did we would clobber the saved TOC
484 * value of the original caller.
485 */
486static void squash_toc_save_inst(const char *name, unsigned long addr)
487{
488 struct ppc64_stub_entry *stub = (struct ppc64_stub_entry *)addr;
489
490 /* Only for calls to _mcount */
491 if (strcmp("_mcount", name) != 0)
492 return;
493
494 stub->jump[2] = PPC_INST_NOP;
495}
496#else
497static void squash_toc_save_inst(const char *name, unsigned long addr) { }
498
499/* without -mprofile-kernel, mcount calls are never early */
500static bool is_early_mcount_callsite(u32 *instruction)
501{
502 return false;
503}
504#endif
505
506/* We expect a noop next: if it is, replace it with instruction to
507 restore r2. */
508static int restore_r2(u32 *instruction, struct module *me)
509{
510 u32 *prev_insn = instruction - 1;
511
512 if (is_early_mcount_callsite(prev_insn))
513 return 1;
514
515 /*
516 * Make sure the branch isn't a sibling call. Sibling calls aren't
517 * "link" branches and they don't return, so they don't need the r2
518 * restore afterwards.
519 */
520 if (!instr_is_relative_link_branch(*prev_insn))
521 return 1;
522
523 if (*instruction != PPC_INST_NOP) {
524 pr_err("%s: Expected nop after call, got %08x at %pS\n",
525 me->name, *instruction, instruction);
526 return 0;
527 }
528 /* ld r2,R2_STACK_OFFSET(r1) */
529 *instruction = PPC_INST_LD_TOC;
530 return 1;
531}
532
533int apply_relocate_add(Elf64_Shdr *sechdrs,
534 const char *strtab,
535 unsigned int symindex,
536 unsigned int relsec,
537 struct module *me)
538{
539 unsigned int i;
540 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
541 Elf64_Sym *sym;
542 unsigned long *location;
543 unsigned long value;
544
545 pr_debug("Applying ADD relocate section %u to %u\n", relsec,
546 sechdrs[relsec].sh_info);
547
548 /* First time we're called, we can fix up .TOC. */
549 if (!me->arch.toc_fixed) {
550 sym = find_dot_toc(sechdrs, strtab, symindex);
551 /* It's theoretically possible that a module doesn't want a
552 * .TOC. so don't fail it just for that. */
553 if (sym)
554 sym->st_value = my_r2(sechdrs, me);
555 me->arch.toc_fixed = true;
556 }
557
558 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
559 /* This is where to make the change */
560 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
561 + rela[i].r_offset;
562 /* This is the symbol it is referring to */
563 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
564 + ELF64_R_SYM(rela[i].r_info);
565
566 pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n",
567 location, (long)ELF64_R_TYPE(rela[i].r_info),
568 strtab + sym->st_name, (unsigned long)sym->st_value,
569 (long)rela[i].r_addend);
570
571 /* `Everything is relative'. */
572 value = sym->st_value + rela[i].r_addend;
573
574 switch (ELF64_R_TYPE(rela[i].r_info)) {
575 case R_PPC64_ADDR32:
576 /* Simply set it */
577 *(u32 *)location = value;
578 break;
579
580 case R_PPC64_ADDR64:
581 /* Simply set it */
582 *(unsigned long *)location = value;
583 break;
584
585 case R_PPC64_TOC:
586 *(unsigned long *)location = my_r2(sechdrs, me);
587 break;
588
589 case R_PPC64_TOC16:
590 /* Subtract TOC pointer */
591 value -= my_r2(sechdrs, me);
592 if (value + 0x8000 > 0xffff) {
593 pr_err("%s: bad TOC16 relocation (0x%lx)\n",
594 me->name, value);
595 return -ENOEXEC;
596 }
597 *((uint16_t *) location)
598 = (*((uint16_t *) location) & ~0xffff)
599 | (value & 0xffff);
600 break;
601
602 case R_PPC64_TOC16_LO:
603 /* Subtract TOC pointer */
604 value -= my_r2(sechdrs, me);
605 *((uint16_t *) location)
606 = (*((uint16_t *) location) & ~0xffff)
607 | (value & 0xffff);
608 break;
609
610 case R_PPC64_TOC16_DS:
611 /* Subtract TOC pointer */
612 value -= my_r2(sechdrs, me);
613 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
614 pr_err("%s: bad TOC16_DS relocation (0x%lx)\n",
615 me->name, value);
616 return -ENOEXEC;
617 }
618 *((uint16_t *) location)
619 = (*((uint16_t *) location) & ~0xfffc)
620 | (value & 0xfffc);
621 break;
622
623 case R_PPC64_TOC16_LO_DS:
624 /* Subtract TOC pointer */
625 value -= my_r2(sechdrs, me);
626 if ((value & 3) != 0) {
627 pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n",
628 me->name, value);
629 return -ENOEXEC;
630 }
631 *((uint16_t *) location)
632 = (*((uint16_t *) location) & ~0xfffc)
633 | (value & 0xfffc);
634 break;
635
636 case R_PPC64_TOC16_HA:
637 /* Subtract TOC pointer */
638 value -= my_r2(sechdrs, me);
639 value = ((value + 0x8000) >> 16);
640 *((uint16_t *) location)
641 = (*((uint16_t *) location) & ~0xffff)
642 | (value & 0xffff);
643 break;
644
645 case R_PPC_REL24:
646 /* FIXME: Handle weak symbols here --RR */
647 if (sym->st_shndx == SHN_UNDEF ||
648 sym->st_shndx == SHN_LIVEPATCH) {
649 /* External: go via stub */
650 value = stub_for_addr(sechdrs, value, me);
651 if (!value)
652 return -ENOENT;
653 if (!restore_r2((u32 *)location + 1, me))
654 return -ENOEXEC;
655
656 squash_toc_save_inst(strtab + sym->st_name, value);
657 } else
658 value += local_entry_offset(sym);
659
660 /* Convert value to relative */
661 value -= (unsigned long)location;
662 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
663 pr_err("%s: REL24 %li out of range!\n",
664 me->name, (long int)value);
665 return -ENOEXEC;
666 }
667
668 /* Only replace bits 2 through 26 */
669 *(uint32_t *)location
670 = (*(uint32_t *)location & ~0x03fffffc)
671 | (value & 0x03fffffc);
672 break;
673
674 case R_PPC64_REL64:
675 /* 64 bits relative (used by features fixups) */
676 *location = value - (unsigned long)location;
677 break;
678
679 case R_PPC64_REL32:
680 /* 32 bits relative (used by relative exception tables) */
681 *(u32 *)location = value - (unsigned long)location;
682 break;
683
684 case R_PPC64_TOCSAVE:
685 /*
686 * Marker reloc indicates we don't have to save r2.
687 * That would only save us one instruction, so ignore
688 * it.
689 */
690 break;
691
692 case R_PPC64_ENTRY:
693 /*
694 * Optimize ELFv2 large code model entry point if
695 * the TOC is within 2GB range of current location.
696 */
697 value = my_r2(sechdrs, me) - (unsigned long)location;
698 if (value + 0x80008000 > 0xffffffff)
699 break;
700 /*
701 * Check for the large code model prolog sequence:
702 * ld r2, ...(r12)
703 * add r2, r2, r12
704 */
705 if ((((uint32_t *)location)[0] & ~0xfffc)
706 != 0xe84c0000)
707 break;
708 if (((uint32_t *)location)[1] != 0x7c426214)
709 break;
710 /*
711 * If found, replace it with:
712 * addis r2, r12, (.TOC.-func)@ha
713 * addi r2, r12, (.TOC.-func)@l
714 */
715 ((uint32_t *)location)[0] = 0x3c4c0000 + PPC_HA(value);
716 ((uint32_t *)location)[1] = 0x38420000 + PPC_LO(value);
717 break;
718
719 case R_PPC64_REL16_HA:
720 /* Subtract location pointer */
721 value -= (unsigned long)location;
722 value = ((value + 0x8000) >> 16);
723 *((uint16_t *) location)
724 = (*((uint16_t *) location) & ~0xffff)
725 | (value & 0xffff);
726 break;
727
728 case R_PPC64_REL16_LO:
729 /* Subtract location pointer */
730 value -= (unsigned long)location;
731 *((uint16_t *) location)
732 = (*((uint16_t *) location) & ~0xffff)
733 | (value & 0xffff);
734 break;
735
736 default:
737 pr_err("%s: Unknown ADD relocation: %lu\n",
738 me->name,
739 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
740 return -ENOEXEC;
741 }
742 }
743
744 return 0;
745}
746
747#ifdef CONFIG_DYNAMIC_FTRACE
748
749#ifdef CC_USING_MPROFILE_KERNEL
750
751#define PACATOC offsetof(struct paca_struct, kernel_toc)
752
753/*
754 * For mprofile-kernel we use a special stub for ftrace_caller() because we
755 * can't rely on r2 containing this module's TOC when we enter the stub.
756 *
757 * That can happen if the function calling us didn't need to use the toc. In
758 * that case it won't have setup r2, and the r2 value will be either the
759 * kernel's toc, or possibly another modules toc.
760 *
761 * To deal with that this stub uses the kernel toc, which is always accessible
762 * via the paca (in r13). The target (ftrace_caller()) is responsible for
763 * saving and restoring the toc before returning.
764 */
765static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
766{
767 struct ppc64_stub_entry *entry;
768 unsigned int i, num_stubs;
769 static u32 stub_insns[] = {
770 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */
771 0x3d8c0000, /* addis r12,r12,<high> */
772 0x398c0000, /* addi r12,r12,<low> */
773 0x7d8903a6, /* mtctr r12 */
774 0x4e800420, /* bctr */
775 };
776 long reladdr;
777
778 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*entry);
779
780 /* Find the next available stub entry */
781 entry = (void *)sechdrs[me->arch.stubs_section].sh_addr;
782 for (i = 0; i < num_stubs && stub_func_addr(entry->funcdata); i++, entry++);
783
784 if (i >= num_stubs) {
785 pr_err("%s: Unable to find a free slot for ftrace stub.\n", me->name);
786 return 0;
787 }
788
789 memcpy(entry->jump, stub_insns, sizeof(stub_insns));
790
791 /* Stub uses address relative to kernel toc (from the paca) */
792 reladdr = (unsigned long)ftrace_caller - kernel_toc_addr();
793 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
794 pr_err("%s: Address of ftrace_caller out of range of kernel_toc.\n", me->name);
795 return 0;
796 }
797
798 entry->jump[1] |= PPC_HA(reladdr);
799 entry->jump[2] |= PPC_LO(reladdr);
800
801 /* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */
802 entry->funcdata = func_desc((unsigned long)ftrace_caller);
803 entry->magic = STUB_MAGIC;
804
805 return (unsigned long)entry;
806}
807#else
808static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
809{
810 return stub_for_addr(sechdrs, (unsigned long)ftrace_caller, me);
811}
812#endif
813
814int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
815{
816 mod->arch.toc = my_r2(sechdrs, mod);
817 mod->arch.tramp = create_ftrace_stub(sechdrs, mod);
818
819 if (!mod->arch.tramp)
820 return -ENOENT;
821
822 return 0;
823}
824#endif
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Kernel module help for PPC64.
3 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
4
5*/
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/module.h>
10#include <linux/elf.h>
11#include <linux/moduleloader.h>
12#include <linux/err.h>
13#include <linux/vmalloc.h>
14#include <linux/ftrace.h>
15#include <linux/bug.h>
16#include <linux/uaccess.h>
17#include <linux/kernel.h>
18#include <asm/module.h>
19#include <asm/firmware.h>
20#include <asm/code-patching.h>
21#include <linux/sort.h>
22#include <asm/setup.h>
23#include <asm/sections.h>
24#include <asm/inst.h>
25
26/* FIXME: We don't do .init separately. To do this, we'd need to have
27 a separate r2 value in the init and core section, and stub between
28 them, too.
29
30 Using a magic allocator which places modules within 32MB solves
31 this, and makes other things simpler. Anton?
32 --RR. */
33
34bool module_elf_check_arch(Elf_Ehdr *hdr)
35{
36 unsigned long abi_level = hdr->e_flags & 0x3;
37
38 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
39 return abi_level == 2;
40 else
41 return abi_level < 2;
42}
43
44#ifdef CONFIG_PPC64_ELF_ABI_V2
45
46static func_desc_t func_desc(unsigned long addr)
47{
48 func_desc_t desc = {
49 .addr = addr,
50 };
51
52 return desc;
53}
54
55/* PowerPC64 specific values for the Elf64_Sym st_other field. */
56#define STO_PPC64_LOCAL_BIT 5
57#define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT)
58#define PPC64_LOCAL_ENTRY_OFFSET(other) \
59 (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)
60
61static unsigned int local_entry_offset(const Elf64_Sym *sym)
62{
63 /* sym->st_other indicates offset to local entry point
64 * (otherwise it will assume r12 is the address of the start
65 * of function and try to derive r2 from it). */
66 return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other);
67}
68#else
69
70static func_desc_t func_desc(unsigned long addr)
71{
72 return *(struct func_desc *)addr;
73}
74static unsigned int local_entry_offset(const Elf64_Sym *sym)
75{
76 return 0;
77}
78
79void *dereference_module_function_descriptor(struct module *mod, void *ptr)
80{
81 if (ptr < (void *)mod->arch.start_opd ||
82 ptr >= (void *)mod->arch.end_opd)
83 return ptr;
84
85 return dereference_function_descriptor(ptr);
86}
87#endif
88
89static unsigned long func_addr(unsigned long addr)
90{
91 return func_desc(addr).addr;
92}
93
94static unsigned long stub_func_addr(func_desc_t func)
95{
96 return func.addr;
97}
98
99#define STUB_MAGIC 0x73747562 /* stub */
100
101/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
102 the kernel itself). But on PPC64, these need to be used for every
103 jump, actually, to reset r2 (TOC+0x8000). */
104struct ppc64_stub_entry {
105 /*
106 * 28 byte jump instruction sequence (7 instructions) that can
107 * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned
108 * with PCREL kernels that use prefix instructions in the stub.
109 */
110 u32 jump[7];
111 /* Used by ftrace to identify stubs */
112 u32 magic;
113 /* Data for the above code */
114 func_desc_t funcdata;
115} __aligned(8);
116
117struct ppc64_got_entry {
118 u64 addr;
119};
120
121/*
122 * PPC64 uses 24 bit jumps, but we need to jump into other modules or
123 * the kernel which may be further. So we jump to a stub.
124 *
125 * Target address and TOC are loaded from function descriptor in the
126 * ppc64_stub_entry.
127 *
128 * r12 is used to generate the target address, which is required for the
129 * ELFv2 global entry point calling convention.
130 *
131 * TOC handling:
132 * - PCREL does not have a TOC.
133 * - ELFv2 non-PCREL just has to save r2, the callee is responsible for
134 * setting its own TOC pointer at the global entry address.
135 * - ELFv1 must load the new TOC pointer from the function descriptor.
136 */
137static u32 ppc64_stub_insns[] = {
138#ifdef CONFIG_PPC_KERNEL_PCREL
139 /* pld r12,addr */
140 PPC_PREFIX_8LS | __PPC_PRFX_R(1),
141 PPC_INST_PLD | ___PPC_RT(_R12),
142#else
143 PPC_RAW_ADDIS(_R11, _R2, 0),
144 PPC_RAW_ADDI(_R11, _R11, 0),
145 /* Save current r2 value in magic place on the stack. */
146 PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET),
147 PPC_RAW_LD(_R12, _R11, 32),
148#ifdef CONFIG_PPC64_ELF_ABI_V1
149 /* Set up new r2 from function descriptor */
150 PPC_RAW_LD(_R2, _R11, 40),
151#endif
152#endif
153 PPC_RAW_MTCTR(_R12),
154 PPC_RAW_BCTR(),
155};
156
157/*
158 * Count how many different r_type relocations (different symbol,
159 * different addend).
160 */
161static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num,
162 unsigned long r_type)
163{
164 unsigned int i, r_info, r_addend, _count_relocs;
165
166 /* FIXME: Only count external ones --RR */
167 _count_relocs = 0;
168 r_info = 0;
169 r_addend = 0;
170 for (i = 0; i < num; i++)
171 /* Only count r_type relocs, others don't need stubs */
172 if (ELF64_R_TYPE(rela[i].r_info) == r_type &&
173 (r_info != ELF64_R_SYM(rela[i].r_info) ||
174 r_addend != rela[i].r_addend)) {
175 _count_relocs++;
176 r_info = ELF64_R_SYM(rela[i].r_info);
177 r_addend = rela[i].r_addend;
178 }
179
180 return _count_relocs;
181}
182
183static int relacmp(const void *_x, const void *_y)
184{
185 const Elf64_Rela *x, *y;
186
187 y = (Elf64_Rela *)_x;
188 x = (Elf64_Rela *)_y;
189
190 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
191 * make the comparison cheaper/faster. It won't affect the sorting or
192 * the counting algorithms' performance
193 */
194 if (x->r_info < y->r_info)
195 return -1;
196 else if (x->r_info > y->r_info)
197 return 1;
198 else if (x->r_addend < y->r_addend)
199 return -1;
200 else if (x->r_addend > y->r_addend)
201 return 1;
202 else
203 return 0;
204}
205
206/* Get size of potential trampolines required. */
207static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
208 const Elf64_Shdr *sechdrs)
209{
210 /* One extra reloc so it's always 0-addr terminated */
211 unsigned long relocs = 1;
212 unsigned i;
213
214 /* Every relocated section... */
215 for (i = 1; i < hdr->e_shnum; i++) {
216 if (sechdrs[i].sh_type == SHT_RELA) {
217 pr_debug("Found relocations in section %u\n", i);
218 pr_debug("Ptr: %p. Number: %Lu\n",
219 (void *)sechdrs[i].sh_addr,
220 sechdrs[i].sh_size / sizeof(Elf64_Rela));
221
222 /* Sort the relocation information based on a symbol and
223 * addend key. This is a stable O(n*log n) complexity
224 * algorithm but it will reduce the complexity of
225 * count_relocs() to linear complexity O(n)
226 */
227 sort((void *)sechdrs[i].sh_addr,
228 sechdrs[i].sh_size / sizeof(Elf64_Rela),
229 sizeof(Elf64_Rela), relacmp, NULL);
230
231 relocs += count_relocs((void *)sechdrs[i].sh_addr,
232 sechdrs[i].sh_size
233 / sizeof(Elf64_Rela),
234 R_PPC_REL24);
235#ifdef CONFIG_PPC_KERNEL_PCREL
236 relocs += count_relocs((void *)sechdrs[i].sh_addr,
237 sechdrs[i].sh_size
238 / sizeof(Elf64_Rela),
239 R_PPC64_REL24_NOTOC);
240#endif
241 }
242 }
243
244#ifdef CONFIG_DYNAMIC_FTRACE
245 /* make the trampoline to the ftrace_caller */
246 relocs++;
247#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
248 /* an additional one for ftrace_regs_caller */
249 relocs++;
250#endif
251#endif
252
253 pr_debug("Looks like a total of %lu stubs, max\n", relocs);
254 return relocs * sizeof(struct ppc64_stub_entry);
255}
256
257#ifdef CONFIG_PPC_KERNEL_PCREL
258static int count_pcpu_relocs(const Elf64_Shdr *sechdrs,
259 const Elf64_Rela *rela, unsigned int num,
260 unsigned int symindex, unsigned int pcpu)
261{
262 unsigned int i, r_info, r_addend, _count_relocs;
263
264 _count_relocs = 0;
265 r_info = 0;
266 r_addend = 0;
267
268 for (i = 0; i < num; i++) {
269 Elf64_Sym *sym;
270
271 /* This is the symbol it is referring to */
272 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
273 + ELF64_R_SYM(rela[i].r_info);
274
275 if (sym->st_shndx == pcpu &&
276 (r_info != ELF64_R_SYM(rela[i].r_info) ||
277 r_addend != rela[i].r_addend)) {
278 _count_relocs++;
279 r_info = ELF64_R_SYM(rela[i].r_info);
280 r_addend = rela[i].r_addend;
281 }
282 }
283
284 return _count_relocs;
285}
286
287/* Get size of potential GOT required. */
288static unsigned long get_got_size(const Elf64_Ehdr *hdr,
289 const Elf64_Shdr *sechdrs,
290 struct module *me)
291{
292 /* One extra reloc so it's always 0-addr terminated */
293 unsigned long relocs = 1;
294 unsigned int i, symindex = 0;
295
296 for (i = 1; i < hdr->e_shnum; i++) {
297 if (sechdrs[i].sh_type == SHT_SYMTAB) {
298 symindex = i;
299 break;
300 }
301 }
302 WARN_ON_ONCE(!symindex);
303
304 /* Every relocated section... */
305 for (i = 1; i < hdr->e_shnum; i++) {
306 if (sechdrs[i].sh_type == SHT_RELA) {
307 pr_debug("Found relocations in section %u\n", i);
308 pr_debug("Ptr: %p. Number: %llu\n", (void *)sechdrs[i].sh_addr,
309 sechdrs[i].sh_size / sizeof(Elf64_Rela));
310
311 /*
312 * Sort the relocation information based on a symbol and
313 * addend key. This is a stable O(n*log n) complexity
314 * algorithm but it will reduce the complexity of
315 * count_relocs() to linear complexity O(n)
316 */
317 sort((void *)sechdrs[i].sh_addr,
318 sechdrs[i].sh_size / sizeof(Elf64_Rela),
319 sizeof(Elf64_Rela), relacmp, NULL);
320
321 relocs += count_relocs((void *)sechdrs[i].sh_addr,
322 sechdrs[i].sh_size
323 / sizeof(Elf64_Rela),
324 R_PPC64_GOT_PCREL34);
325
326 /*
327 * Percpu data access typically gets linked with
328 * REL34 relocations, but the percpu section gets
329 * moved at load time and requires that to be
330 * converted to GOT linkage.
331 */
332 if (IS_ENABLED(CONFIG_SMP) && symindex)
333 relocs += count_pcpu_relocs(sechdrs,
334 (void *)sechdrs[i].sh_addr,
335 sechdrs[i].sh_size
336 / sizeof(Elf64_Rela),
337 symindex, me->arch.pcpu_section);
338 }
339 }
340
341 pr_debug("Looks like a total of %lu GOT entries, max\n", relocs);
342 return relocs * sizeof(struct ppc64_got_entry);
343}
344#else /* CONFIG_PPC_KERNEL_PCREL */
345
346/* Still needed for ELFv2, for .TOC. */
347static void dedotify_versions(struct modversion_info *vers,
348 unsigned long size)
349{
350 struct modversion_info *end;
351
352 for (end = (void *)vers + size; vers < end; vers++)
353 if (vers->name[0] == '.') {
354 memmove(vers->name, vers->name+1, strlen(vers->name));
355 }
356}
357
358/*
359 * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC.
360 * seem to be defined (value set later).
361 */
362static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
363{
364 unsigned int i;
365
366 for (i = 1; i < numsyms; i++) {
367 if (syms[i].st_shndx == SHN_UNDEF) {
368 char *name = strtab + syms[i].st_name;
369 if (name[0] == '.') {
370 if (strcmp(name+1, "TOC.") == 0)
371 syms[i].st_shndx = SHN_ABS;
372 syms[i].st_name++;
373 }
374 }
375 }
376}
377
378static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs,
379 const char *strtab,
380 unsigned int symindex)
381{
382 unsigned int i, numsyms;
383 Elf64_Sym *syms;
384
385 syms = (Elf64_Sym *)sechdrs[symindex].sh_addr;
386 numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym);
387
388 for (i = 1; i < numsyms; i++) {
389 if (syms[i].st_shndx == SHN_ABS
390 && strcmp(strtab + syms[i].st_name, "TOC.") == 0)
391 return &syms[i];
392 }
393 return NULL;
394}
395#endif /* CONFIG_PPC_KERNEL_PCREL */
396
397bool module_init_section(const char *name)
398{
399 /* We don't handle .init for the moment: always return false. */
400 return false;
401}
402
403int module_frob_arch_sections(Elf64_Ehdr *hdr,
404 Elf64_Shdr *sechdrs,
405 char *secstrings,
406 struct module *me)
407{
408 unsigned int i;
409
410 /* Find .toc and .stubs sections, symtab and strtab */
411 for (i = 1; i < hdr->e_shnum; i++) {
412 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
413 me->arch.stubs_section = i;
414#ifdef CONFIG_PPC_KERNEL_PCREL
415 else if (strcmp(secstrings + sechdrs[i].sh_name, ".data..percpu") == 0)
416 me->arch.pcpu_section = i;
417 else if (strcmp(secstrings + sechdrs[i].sh_name, ".mygot") == 0) {
418 me->arch.got_section = i;
419 if (sechdrs[i].sh_addralign < 8)
420 sechdrs[i].sh_addralign = 8;
421 }
422#else
423 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) {
424 me->arch.toc_section = i;
425 if (sechdrs[i].sh_addralign < 8)
426 sechdrs[i].sh_addralign = 8;
427 }
428 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
429 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
430 sechdrs[i].sh_size);
431
432 if (sechdrs[i].sh_type == SHT_SYMTAB)
433 dedotify((void *)hdr + sechdrs[i].sh_offset,
434 sechdrs[i].sh_size / sizeof(Elf64_Sym),
435 (void *)hdr
436 + sechdrs[sechdrs[i].sh_link].sh_offset);
437#endif
438 }
439
440 if (!me->arch.stubs_section) {
441 pr_err("%s: doesn't contain .stubs.\n", me->name);
442 return -ENOEXEC;
443 }
444
445#ifdef CONFIG_PPC_KERNEL_PCREL
446 if (!me->arch.got_section) {
447 pr_err("%s: doesn't contain .mygot.\n", me->name);
448 return -ENOEXEC;
449 }
450
451 /* Override the got size */
452 sechdrs[me->arch.got_section].sh_size = get_got_size(hdr, sechdrs, me);
453#else
454 /* If we don't have a .toc, just use .stubs. We need to set r2
455 to some reasonable value in case the module calls out to
456 other functions via a stub, or if a function pointer escapes
457 the module by some means. */
458 if (!me->arch.toc_section)
459 me->arch.toc_section = me->arch.stubs_section;
460#endif
461
462 /* Override the stubs size */
463 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
464
465 return 0;
466}
467
468#if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)
469
470static u32 stub_insns[] = {
471#ifdef CONFIG_PPC_KERNEL_PCREL
472 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)),
473 PPC_RAW_NOP(), /* align the prefix insn */
474 /* paddi r12,r12,addr */
475 PPC_PREFIX_MLS | __PPC_PRFX_R(0),
476 PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12),
477 PPC_RAW_MTCTR(_R12),
478 PPC_RAW_BCTR(),
479#else
480 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
481 PPC_RAW_ADDIS(_R12, _R12, 0),
482 PPC_RAW_ADDI(_R12, _R12, 0),
483 PPC_RAW_MTCTR(_R12),
484 PPC_RAW_BCTR(),
485#endif
486};
487
488/*
489 * For mprofile-kernel we use a special stub for ftrace_caller() because we
490 * can't rely on r2 containing this module's TOC when we enter the stub.
491 *
492 * That can happen if the function calling us didn't need to use the toc. In
493 * that case it won't have setup r2, and the r2 value will be either the
494 * kernel's toc, or possibly another modules toc.
495 *
496 * To deal with that this stub uses the kernel toc, which is always accessible
497 * via the paca (in r13). The target (ftrace_caller()) is responsible for
498 * saving and restoring the toc before returning.
499 */
500static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
501 unsigned long addr,
502 struct module *me)
503{
504 long reladdr;
505
506 if ((unsigned long)entry->jump % 8 != 0) {
507 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
508 return 0;
509 }
510
511 BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump));
512 memcpy(entry->jump, stub_insns, sizeof(stub_insns));
513
514 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
515 /* Stub uses address relative to kernel base (from the paca) */
516 reladdr = addr - local_paca->kernelbase;
517 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
518 pr_err("%s: Address of %ps out of range of 34-bit relative address.\n",
519 me->name, (void *)addr);
520 return 0;
521 }
522
523 entry->jump[2] |= IMM_H18(reladdr);
524 entry->jump[3] |= IMM_L(reladdr);
525 } else {
526 /* Stub uses address relative to kernel toc (from the paca) */
527 reladdr = addr - kernel_toc_addr();
528 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
529 pr_err("%s: Address of %ps out of range of kernel_toc.\n",
530 me->name, (void *)addr);
531 return 0;
532 }
533
534 entry->jump[1] |= PPC_HA(reladdr);
535 entry->jump[2] |= PPC_LO(reladdr);
536 }
537
538 /* Even though we don't use funcdata in the stub, it's needed elsewhere. */
539 entry->funcdata = func_desc(addr);
540 entry->magic = STUB_MAGIC;
541
542 return 1;
543}
544
545static bool is_mprofile_ftrace_call(const char *name)
546{
547 if (!strcmp("_mcount", name))
548 return true;
549#ifdef CONFIG_DYNAMIC_FTRACE
550 if (!strcmp("ftrace_caller", name))
551 return true;
552#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
553 if (!strcmp("ftrace_regs_caller", name))
554 return true;
555#endif
556#endif
557
558 return false;
559}
560#else
561static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
562 unsigned long addr,
563 struct module *me)
564{
565 return 0;
566}
567
568static bool is_mprofile_ftrace_call(const char *name)
569{
570 return false;
571}
572#endif
573
574/*
575 * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
576 * value maximum span in an instruction which uses a signed offset). Round down
577 * to a 256 byte boundary for the odd case where we are setting up r2 without a
578 * .toc section.
579 */
580static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me)
581{
582#ifndef CONFIG_PPC_KERNEL_PCREL
583 return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
584#else
585 return -1;
586#endif
587}
588
589/* Patch stub to reference function and correct r2 value. */
590static inline int create_stub(const Elf64_Shdr *sechdrs,
591 struct ppc64_stub_entry *entry,
592 unsigned long addr,
593 struct module *me,
594 const char *name)
595{
596 long reladdr;
597 func_desc_t desc;
598 int i;
599
600 if (is_mprofile_ftrace_call(name))
601 return create_ftrace_stub(entry, addr, me);
602
603 if ((unsigned long)entry->jump % 8 != 0) {
604 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
605 return 0;
606 }
607
608 BUILD_BUG_ON(sizeof(ppc64_stub_insns) > sizeof(entry->jump));
609 for (i = 0; i < ARRAY_SIZE(ppc64_stub_insns); i++) {
610 if (patch_instruction(&entry->jump[i],
611 ppc_inst(ppc64_stub_insns[i])))
612 return 0;
613 }
614
615 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
616 /* Stub uses address relative to itself! */
617 reladdr = 0 + offsetof(struct ppc64_stub_entry, funcdata);
618 BUILD_BUG_ON(reladdr != 32);
619 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
620 pr_err("%s: Address of %p out of range of 34-bit relative address.\n",
621 me->name, (void *)reladdr);
622 return 0;
623 }
624 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
625
626 /* May not even need this if we're relative to 0 */
627 if (patch_instruction(&entry->jump[0],
628 ppc_inst_prefix(entry->jump[0] | IMM_H18(reladdr),
629 entry->jump[1] | IMM_L(reladdr))))
630 return 0;
631
632 } else {
633 /* Stub uses address relative to r2. */
634 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
635 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
636 pr_err("%s: Address %p of stub out of range of %p.\n",
637 me->name, (void *)reladdr, (void *)my_r2);
638 return 0;
639 }
640 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
641
642 if (patch_instruction(&entry->jump[0],
643 ppc_inst(entry->jump[0] | PPC_HA(reladdr))))
644 return 0;
645
646 if (patch_instruction(&entry->jump[1],
647 ppc_inst(entry->jump[1] | PPC_LO(reladdr))))
648 return 0;
649 }
650
651 // func_desc_t is 8 bytes if ABIv2, else 16 bytes
652 desc = func_desc(addr);
653 for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) {
654 if (patch_instruction(((u32 *)&entry->funcdata) + i,
655 ppc_inst(((u32 *)(&desc))[i])))
656 return 0;
657 }
658
659 if (patch_instruction(&entry->magic, ppc_inst(STUB_MAGIC)))
660 return 0;
661
662 return 1;
663}
664
665/* Create stub to jump to function described in this OPD/ptr: we need the
666 stub to set up the TOC ptr (r2) for the function. */
667static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
668 unsigned long addr,
669 struct module *me,
670 const char *name)
671{
672 struct ppc64_stub_entry *stubs;
673 unsigned int i, num_stubs;
674
675 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
676
677 /* Find this stub, or if that fails, the next avail. entry */
678 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
679 for (i = 0; stub_func_addr(stubs[i].funcdata); i++) {
680 if (WARN_ON(i >= num_stubs))
681 return 0;
682
683 if (stub_func_addr(stubs[i].funcdata) == func_addr(addr))
684 return (unsigned long)&stubs[i];
685 }
686
687 if (!create_stub(sechdrs, &stubs[i], addr, me, name))
688 return 0;
689
690 return (unsigned long)&stubs[i];
691}
692
693#ifdef CONFIG_PPC_KERNEL_PCREL
694/* Create GOT to load the location described in this ptr */
695static unsigned long got_for_addr(const Elf64_Shdr *sechdrs,
696 unsigned long addr,
697 struct module *me,
698 const char *name)
699{
700 struct ppc64_got_entry *got;
701 unsigned int i, num_got;
702
703 if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
704 return addr;
705
706 num_got = sechdrs[me->arch.got_section].sh_size / sizeof(*got);
707
708 /* Find this stub, or if that fails, the next avail. entry */
709 got = (void *)sechdrs[me->arch.got_section].sh_addr;
710 for (i = 0; got[i].addr; i++) {
711 if (WARN_ON(i >= num_got))
712 return 0;
713
714 if (got[i].addr == addr)
715 return (unsigned long)&got[i];
716 }
717
718 got[i].addr = addr;
719
720 return (unsigned long)&got[i];
721}
722#endif
723
724/* We expect a noop next: if it is, replace it with instruction to
725 restore r2. */
726static int restore_r2(const char *name, u32 *instruction, struct module *me)
727{
728 u32 *prev_insn = instruction - 1;
729 u32 insn_val = *instruction;
730
731 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
732 return 0;
733
734 if (is_mprofile_ftrace_call(name))
735 return 0;
736
737 /*
738 * Make sure the branch isn't a sibling call. Sibling calls aren't
739 * "link" branches and they don't return, so they don't need the r2
740 * restore afterwards.
741 */
742 if (!instr_is_relative_link_branch(ppc_inst(*prev_insn)))
743 return 0;
744
745 /*
746 * For livepatch, the restore r2 instruction might have already been
747 * written previously, if the referenced symbol is in a previously
748 * unloaded module which is now being loaded again. In that case, skip
749 * the warning and the instruction write.
750 */
751 if (insn_val == PPC_INST_LD_TOC)
752 return 0;
753
754 if (insn_val != PPC_RAW_NOP()) {
755 pr_err("%s: Expected nop after call, got %08x at %pS\n",
756 me->name, insn_val, instruction);
757 return -ENOEXEC;
758 }
759
760 /* ld r2,R2_STACK_OFFSET(r1) */
761 return patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC));
762}
763
764int apply_relocate_add(Elf64_Shdr *sechdrs,
765 const char *strtab,
766 unsigned int symindex,
767 unsigned int relsec,
768 struct module *me)
769{
770 unsigned int i;
771 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
772 Elf64_Sym *sym;
773 unsigned long *location;
774 unsigned long value;
775
776 pr_debug("Applying ADD relocate section %u to %u\n", relsec,
777 sechdrs[relsec].sh_info);
778
779#ifndef CONFIG_PPC_KERNEL_PCREL
780 /* First time we're called, we can fix up .TOC. */
781 if (!me->arch.toc_fixed) {
782 sym = find_dot_toc(sechdrs, strtab, symindex);
783 /* It's theoretically possible that a module doesn't want a
784 * .TOC. so don't fail it just for that. */
785 if (sym)
786 sym->st_value = my_r2(sechdrs, me);
787 me->arch.toc_fixed = true;
788 }
789#endif
790 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
791 /* This is where to make the change */
792 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
793 + rela[i].r_offset;
794 /* This is the symbol it is referring to */
795 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
796 + ELF64_R_SYM(rela[i].r_info);
797
798 pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n",
799 location, (long)ELF64_R_TYPE(rela[i].r_info),
800 strtab + sym->st_name, (unsigned long)sym->st_value,
801 (long)rela[i].r_addend);
802
803 /* `Everything is relative'. */
804 value = sym->st_value + rela[i].r_addend;
805
806 switch (ELF64_R_TYPE(rela[i].r_info)) {
807 case R_PPC64_ADDR32:
808 /* Simply set it */
809 *(u32 *)location = value;
810 break;
811
812 case R_PPC64_ADDR64:
813 /* Simply set it */
814 *(unsigned long *)location = value;
815 break;
816
817#ifndef CONFIG_PPC_KERNEL_PCREL
818 case R_PPC64_TOC:
819 *(unsigned long *)location = my_r2(sechdrs, me);
820 break;
821
822 case R_PPC64_TOC16:
823 /* Subtract TOC pointer */
824 value -= my_r2(sechdrs, me);
825 if (value + 0x8000 > 0xffff) {
826 pr_err("%s: bad TOC16 relocation (0x%lx)\n",
827 me->name, value);
828 return -ENOEXEC;
829 }
830 *((uint16_t *) location)
831 = (*((uint16_t *) location) & ~0xffff)
832 | (value & 0xffff);
833 break;
834
835 case R_PPC64_TOC16_LO:
836 /* Subtract TOC pointer */
837 value -= my_r2(sechdrs, me);
838 *((uint16_t *) location)
839 = (*((uint16_t *) location) & ~0xffff)
840 | (value & 0xffff);
841 break;
842
843 case R_PPC64_TOC16_DS:
844 /* Subtract TOC pointer */
845 value -= my_r2(sechdrs, me);
846 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
847 pr_err("%s: bad TOC16_DS relocation (0x%lx)\n",
848 me->name, value);
849 return -ENOEXEC;
850 }
851 *((uint16_t *) location)
852 = (*((uint16_t *) location) & ~0xfffc)
853 | (value & 0xfffc);
854 break;
855
856 case R_PPC64_TOC16_LO_DS:
857 /* Subtract TOC pointer */
858 value -= my_r2(sechdrs, me);
859 if ((value & 3) != 0) {
860 pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n",
861 me->name, value);
862 return -ENOEXEC;
863 }
864 *((uint16_t *) location)
865 = (*((uint16_t *) location) & ~0xfffc)
866 | (value & 0xfffc);
867 break;
868
869 case R_PPC64_TOC16_HA:
870 /* Subtract TOC pointer */
871 value -= my_r2(sechdrs, me);
872 value = ((value + 0x8000) >> 16);
873 *((uint16_t *) location)
874 = (*((uint16_t *) location) & ~0xffff)
875 | (value & 0xffff);
876 break;
877#endif
878
879 case R_PPC_REL24:
880#ifdef CONFIG_PPC_KERNEL_PCREL
881 /* PCREL still generates REL24 for mcount */
882 case R_PPC64_REL24_NOTOC:
883#endif
884 /* FIXME: Handle weak symbols here --RR */
885 if (sym->st_shndx == SHN_UNDEF ||
886 sym->st_shndx == SHN_LIVEPATCH) {
887 /* External: go via stub */
888 value = stub_for_addr(sechdrs, value, me,
889 strtab + sym->st_name);
890 if (!value)
891 return -ENOENT;
892 if (restore_r2(strtab + sym->st_name,
893 (u32 *)location + 1, me))
894 return -ENOEXEC;
895 } else
896 value += local_entry_offset(sym);
897
898 /* Convert value to relative */
899 value -= (unsigned long)location;
900 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
901 pr_err("%s: REL24 %li out of range!\n",
902 me->name, (long int)value);
903 return -ENOEXEC;
904 }
905
906 /* Only replace bits 2 through 26 */
907 value = (*(uint32_t *)location & ~PPC_LI_MASK) | PPC_LI(value);
908
909 if (patch_instruction((u32 *)location, ppc_inst(value)))
910 return -EFAULT;
911
912 break;
913
914 case R_PPC64_REL64:
915 /* 64 bits relative (used by features fixups) */
916 *location = value - (unsigned long)location;
917 break;
918
919 case R_PPC64_REL32:
920 /* 32 bits relative (used by relative exception tables) */
921 /* Convert value to relative */
922 value -= (unsigned long)location;
923 if (value + 0x80000000 > 0xffffffff) {
924 pr_err("%s: REL32 %li out of range!\n",
925 me->name, (long int)value);
926 return -ENOEXEC;
927 }
928 *(u32 *)location = value;
929 break;
930
931#ifdef CONFIG_PPC_KERNEL_PCREL
932 case R_PPC64_PCREL34: {
933 unsigned long absvalue = value;
934
935 /* Convert value to relative */
936 value -= (unsigned long)location;
937
938 if (value + 0x200000000 > 0x3ffffffff) {
939 if (sym->st_shndx != me->arch.pcpu_section) {
940 pr_err("%s: REL34 %li out of range!\n",
941 me->name, (long)value);
942 return -ENOEXEC;
943 }
944
945 /*
946 * per-cpu section is special cased because
947 * it is moved during loading, so has to be
948 * converted to use GOT.
949 */
950 value = got_for_addr(sechdrs, absvalue, me,
951 strtab + sym->st_name);
952 if (!value)
953 return -ENOENT;
954 value -= (unsigned long)location;
955
956 /* Turn pla into pld */
957 if (patch_instruction((u32 *)location,
958 ppc_inst_prefix((*(u32 *)location & ~0x02000000),
959 (*((u32 *)location + 1) & ~0xf8000000) | 0xe4000000)))
960 return -EFAULT;
961 }
962
963 if (patch_instruction((u32 *)location,
964 ppc_inst_prefix((*(u32 *)location & ~0x3ffff) | IMM_H18(value),
965 (*((u32 *)location + 1) & ~0xffff) | IMM_L(value))))
966 return -EFAULT;
967
968 break;
969 }
970
971#else
972 case R_PPC64_TOCSAVE:
973 /*
974 * Marker reloc indicates we don't have to save r2.
975 * That would only save us one instruction, so ignore
976 * it.
977 */
978 break;
979#endif
980
981 case R_PPC64_ENTRY:
982 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
983 break;
984
985 /*
986 * Optimize ELFv2 large code model entry point if
987 * the TOC is within 2GB range of current location.
988 */
989 value = my_r2(sechdrs, me) - (unsigned long)location;
990 if (value + 0x80008000 > 0xffffffff)
991 break;
992 /*
993 * Check for the large code model prolog sequence:
994 * ld r2, ...(r12)
995 * add r2, r2, r12
996 */
997 if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0))
998 break;
999 if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12))
1000 break;
1001 /*
1002 * If found, replace it with:
1003 * addis r2, r12, (.TOC.-func)@ha
1004 * addi r2, r2, (.TOC.-func)@l
1005 */
1006 ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value));
1007 ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value));
1008 break;
1009
1010 case R_PPC64_REL16_HA:
1011 /* Subtract location pointer */
1012 value -= (unsigned long)location;
1013 value = ((value + 0x8000) >> 16);
1014 *((uint16_t *) location)
1015 = (*((uint16_t *) location) & ~0xffff)
1016 | (value & 0xffff);
1017 break;
1018
1019 case R_PPC64_REL16_LO:
1020 /* Subtract location pointer */
1021 value -= (unsigned long)location;
1022 *((uint16_t *) location)
1023 = (*((uint16_t *) location) & ~0xffff)
1024 | (value & 0xffff);
1025 break;
1026
1027#ifdef CONFIG_PPC_KERNEL_PCREL
1028 case R_PPC64_GOT_PCREL34:
1029 value = got_for_addr(sechdrs, value, me,
1030 strtab + sym->st_name);
1031 if (!value)
1032 return -ENOENT;
1033 value -= (unsigned long)location;
1034 ((uint32_t *)location)[0] = (((uint32_t *)location)[0] & ~0x3ffff) |
1035 ((value >> 16) & 0x3ffff);
1036 ((uint32_t *)location)[1] = (((uint32_t *)location)[1] & ~0xffff) |
1037 (value & 0xffff);
1038 break;
1039#endif
1040
1041 default:
1042 pr_err("%s: Unknown ADD relocation: %lu\n",
1043 me->name,
1044 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
1045 return -ENOEXEC;
1046 }
1047 }
1048
1049 return 0;
1050}
1051
1052#ifdef CONFIG_DYNAMIC_FTRACE
1053int module_trampoline_target(struct module *mod, unsigned long addr,
1054 unsigned long *target)
1055{
1056 struct ppc64_stub_entry *stub;
1057 func_desc_t funcdata;
1058 u32 magic;
1059
1060 if (!within_module_core(addr, mod)) {
1061 pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
1062 return -EFAULT;
1063 }
1064
1065 stub = (struct ppc64_stub_entry *)addr;
1066
1067 if (copy_from_kernel_nofault(&magic, &stub->magic,
1068 sizeof(magic))) {
1069 pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
1070 return -EFAULT;
1071 }
1072
1073 if (magic != STUB_MAGIC) {
1074 pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
1075 return -EFAULT;
1076 }
1077
1078 if (copy_from_kernel_nofault(&funcdata, &stub->funcdata,
1079 sizeof(funcdata))) {
1080 pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
1081 return -EFAULT;
1082 }
1083
1084 *target = stub_func_addr(funcdata);
1085
1086 return 0;
1087}
1088
1089int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
1090{
1091 mod->arch.tramp = stub_for_addr(sechdrs,
1092 (unsigned long)ftrace_caller,
1093 mod,
1094 "ftrace_caller");
1095#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1096 mod->arch.tramp_regs = stub_for_addr(sechdrs,
1097 (unsigned long)ftrace_regs_caller,
1098 mod,
1099 "ftrace_regs_caller");
1100 if (!mod->arch.tramp_regs)
1101 return -ENOENT;
1102#endif
1103
1104 if (!mod->arch.tramp)
1105 return -ENOENT;
1106
1107 return 0;
1108}
1109#endif