Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Kernel module help for PPC64.
3 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
4
5*/
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/module.h>
10#include <linux/elf.h>
11#include <linux/moduleloader.h>
12#include <linux/err.h>
13#include <linux/vmalloc.h>
14#include <linux/ftrace.h>
15#include <linux/bug.h>
16#include <linux/uaccess.h>
17#include <linux/kernel.h>
18#include <asm/module.h>
19#include <asm/firmware.h>
20#include <asm/code-patching.h>
21#include <linux/sort.h>
22#include <asm/setup.h>
23#include <asm/sections.h>
24#include <asm/inst.h>
25
26/* FIXME: We don't do .init separately. To do this, we'd need to have
27 a separate r2 value in the init and core section, and stub between
28 them, too.
29
30 Using a magic allocator which places modules within 32MB solves
31 this, and makes other things simpler. Anton?
32 --RR. */
33
34bool module_elf_check_arch(Elf_Ehdr *hdr)
35{
36 unsigned long abi_level = hdr->e_flags & 0x3;
37
38 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
39 return abi_level == 2;
40 else
41 return abi_level < 2;
42}
43
44#ifdef CONFIG_PPC64_ELF_ABI_V2
45
46static func_desc_t func_desc(unsigned long addr)
47{
48 func_desc_t desc = {
49 .addr = addr,
50 };
51
52 return desc;
53}
54
55/* PowerPC64 specific values for the Elf64_Sym st_other field. */
56#define STO_PPC64_LOCAL_BIT 5
57#define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT)
58#define PPC64_LOCAL_ENTRY_OFFSET(other) \
59 (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)
60
61static unsigned int local_entry_offset(const Elf64_Sym *sym)
62{
63 /* sym->st_other indicates offset to local entry point
64 * (otherwise it will assume r12 is the address of the start
65 * of function and try to derive r2 from it). */
66 return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other);
67}
68#else
69
70static func_desc_t func_desc(unsigned long addr)
71{
72 return *(struct func_desc *)addr;
73}
74static unsigned int local_entry_offset(const Elf64_Sym *sym)
75{
76 return 0;
77}
78
79void *dereference_module_function_descriptor(struct module *mod, void *ptr)
80{
81 if (ptr < (void *)mod->arch.start_opd ||
82 ptr >= (void *)mod->arch.end_opd)
83 return ptr;
84
85 return dereference_function_descriptor(ptr);
86}
87#endif
88
89static unsigned long func_addr(unsigned long addr)
90{
91 return func_desc(addr).addr;
92}
93
94static unsigned long stub_func_addr(func_desc_t func)
95{
96 return func.addr;
97}
98
99#define STUB_MAGIC 0x73747562 /* stub */
100
101/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
102 the kernel itself). But on PPC64, these need to be used for every
103 jump, actually, to reset r2 (TOC+0x8000). */
104struct ppc64_stub_entry {
105 /*
106 * 28 byte jump instruction sequence (7 instructions) that can
107 * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned
108 * with PCREL kernels that use prefix instructions in the stub.
109 */
110 u32 jump[7];
111 /* Used by ftrace to identify stubs */
112 u32 magic;
113 /* Data for the above code */
114 func_desc_t funcdata;
115} __aligned(8);
116
117struct ppc64_got_entry {
118 u64 addr;
119};
120
121/*
122 * PPC64 uses 24 bit jumps, but we need to jump into other modules or
123 * the kernel which may be further. So we jump to a stub.
124 *
125 * Target address and TOC are loaded from function descriptor in the
126 * ppc64_stub_entry.
127 *
128 * r12 is used to generate the target address, which is required for the
129 * ELFv2 global entry point calling convention.
130 *
131 * TOC handling:
132 * - PCREL does not have a TOC.
133 * - ELFv2 non-PCREL just has to save r2, the callee is responsible for
134 * setting its own TOC pointer at the global entry address.
135 * - ELFv1 must load the new TOC pointer from the function descriptor.
136 */
137static u32 ppc64_stub_insns[] = {
138#ifdef CONFIG_PPC_KERNEL_PCREL
139 /* pld r12,addr */
140 PPC_PREFIX_8LS | __PPC_PRFX_R(1),
141 PPC_INST_PLD | ___PPC_RT(_R12),
142#else
143 PPC_RAW_ADDIS(_R11, _R2, 0),
144 PPC_RAW_ADDI(_R11, _R11, 0),
145 /* Save current r2 value in magic place on the stack. */
146 PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET),
147 PPC_RAW_LD(_R12, _R11, 32),
148#ifdef CONFIG_PPC64_ELF_ABI_V1
149 /* Set up new r2 from function descriptor */
150 PPC_RAW_LD(_R2, _R11, 40),
151#endif
152#endif
153 PPC_RAW_MTCTR(_R12),
154 PPC_RAW_BCTR(),
155};
156
157/*
158 * Count how many different r_type relocations (different symbol,
159 * different addend).
160 */
161static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num,
162 unsigned long r_type)
163{
164 unsigned int i, r_info, r_addend, _count_relocs;
165
166 /* FIXME: Only count external ones --RR */
167 _count_relocs = 0;
168 r_info = 0;
169 r_addend = 0;
170 for (i = 0; i < num; i++)
171 /* Only count r_type relocs, others don't need stubs */
172 if (ELF64_R_TYPE(rela[i].r_info) == r_type &&
173 (r_info != ELF64_R_SYM(rela[i].r_info) ||
174 r_addend != rela[i].r_addend)) {
175 _count_relocs++;
176 r_info = ELF64_R_SYM(rela[i].r_info);
177 r_addend = rela[i].r_addend;
178 }
179
180 return _count_relocs;
181}
182
183static int relacmp(const void *_x, const void *_y)
184{
185 const Elf64_Rela *x, *y;
186
187 y = (Elf64_Rela *)_x;
188 x = (Elf64_Rela *)_y;
189
190 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
191 * make the comparison cheaper/faster. It won't affect the sorting or
192 * the counting algorithms' performance
193 */
194 if (x->r_info < y->r_info)
195 return -1;
196 else if (x->r_info > y->r_info)
197 return 1;
198 else if (x->r_addend < y->r_addend)
199 return -1;
200 else if (x->r_addend > y->r_addend)
201 return 1;
202 else
203 return 0;
204}
205
206/* Get size of potential trampolines required. */
207static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
208 const Elf64_Shdr *sechdrs)
209{
210 /* One extra reloc so it's always 0-addr terminated */
211 unsigned long relocs = 1;
212 unsigned i;
213
214 /* Every relocated section... */
215 for (i = 1; i < hdr->e_shnum; i++) {
216 if (sechdrs[i].sh_type == SHT_RELA) {
217 pr_debug("Found relocations in section %u\n", i);
218 pr_debug("Ptr: %p. Number: %Lu\n",
219 (void *)sechdrs[i].sh_addr,
220 sechdrs[i].sh_size / sizeof(Elf64_Rela));
221
222 /* Sort the relocation information based on a symbol and
223 * addend key. This is a stable O(n*log n) complexity
224 * algorithm but it will reduce the complexity of
225 * count_relocs() to linear complexity O(n)
226 */
227 sort((void *)sechdrs[i].sh_addr,
228 sechdrs[i].sh_size / sizeof(Elf64_Rela),
229 sizeof(Elf64_Rela), relacmp, NULL);
230
231 relocs += count_relocs((void *)sechdrs[i].sh_addr,
232 sechdrs[i].sh_size
233 / sizeof(Elf64_Rela),
234 R_PPC_REL24);
235#ifdef CONFIG_PPC_KERNEL_PCREL
236 relocs += count_relocs((void *)sechdrs[i].sh_addr,
237 sechdrs[i].sh_size
238 / sizeof(Elf64_Rela),
239 R_PPC64_REL24_NOTOC);
240#endif
241 }
242 }
243
244#ifdef CONFIG_DYNAMIC_FTRACE
245 /* make the trampoline to the ftrace_caller */
246 relocs++;
247#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
248 /* an additional one for ftrace_regs_caller */
249 relocs++;
250#endif
251#endif
252
253 pr_debug("Looks like a total of %lu stubs, max\n", relocs);
254 return relocs * sizeof(struct ppc64_stub_entry);
255}
256
257#ifdef CONFIG_PPC_KERNEL_PCREL
258static int count_pcpu_relocs(const Elf64_Shdr *sechdrs,
259 const Elf64_Rela *rela, unsigned int num,
260 unsigned int symindex, unsigned int pcpu)
261{
262 unsigned int i, r_info, r_addend, _count_relocs;
263
264 _count_relocs = 0;
265 r_info = 0;
266 r_addend = 0;
267
268 for (i = 0; i < num; i++) {
269 Elf64_Sym *sym;
270
271 /* This is the symbol it is referring to */
272 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
273 + ELF64_R_SYM(rela[i].r_info);
274
275 if (sym->st_shndx == pcpu &&
276 (r_info != ELF64_R_SYM(rela[i].r_info) ||
277 r_addend != rela[i].r_addend)) {
278 _count_relocs++;
279 r_info = ELF64_R_SYM(rela[i].r_info);
280 r_addend = rela[i].r_addend;
281 }
282 }
283
284 return _count_relocs;
285}
286
287/* Get size of potential GOT required. */
288static unsigned long get_got_size(const Elf64_Ehdr *hdr,
289 const Elf64_Shdr *sechdrs,
290 struct module *me)
291{
292 /* One extra reloc so it's always 0-addr terminated */
293 unsigned long relocs = 1;
294 unsigned int i, symindex = 0;
295
296 for (i = 1; i < hdr->e_shnum; i++) {
297 if (sechdrs[i].sh_type == SHT_SYMTAB) {
298 symindex = i;
299 break;
300 }
301 }
302 WARN_ON_ONCE(!symindex);
303
304 /* Every relocated section... */
305 for (i = 1; i < hdr->e_shnum; i++) {
306 if (sechdrs[i].sh_type == SHT_RELA) {
307 pr_debug("Found relocations in section %u\n", i);
308 pr_debug("Ptr: %p. Number: %llu\n", (void *)sechdrs[i].sh_addr,
309 sechdrs[i].sh_size / sizeof(Elf64_Rela));
310
311 /*
312 * Sort the relocation information based on a symbol and
313 * addend key. This is a stable O(n*log n) complexity
314 * algorithm but it will reduce the complexity of
315 * count_relocs() to linear complexity O(n)
316 */
317 sort((void *)sechdrs[i].sh_addr,
318 sechdrs[i].sh_size / sizeof(Elf64_Rela),
319 sizeof(Elf64_Rela), relacmp, NULL);
320
321 relocs += count_relocs((void *)sechdrs[i].sh_addr,
322 sechdrs[i].sh_size
323 / sizeof(Elf64_Rela),
324 R_PPC64_GOT_PCREL34);
325
326 /*
327 * Percpu data access typically gets linked with
328 * REL34 relocations, but the percpu section gets
329 * moved at load time and requires that to be
330 * converted to GOT linkage.
331 */
332 if (IS_ENABLED(CONFIG_SMP) && symindex)
333 relocs += count_pcpu_relocs(sechdrs,
334 (void *)sechdrs[i].sh_addr,
335 sechdrs[i].sh_size
336 / sizeof(Elf64_Rela),
337 symindex, me->arch.pcpu_section);
338 }
339 }
340
341 pr_debug("Looks like a total of %lu GOT entries, max\n", relocs);
342 return relocs * sizeof(struct ppc64_got_entry);
343}
344#else /* CONFIG_PPC_KERNEL_PCREL */
345
346/* Still needed for ELFv2, for .TOC. */
347static void dedotify_versions(struct modversion_info *vers,
348 unsigned long size)
349{
350 struct modversion_info *end;
351
352 for (end = (void *)vers + size; vers < end; vers++)
353 if (vers->name[0] == '.') {
354 memmove(vers->name, vers->name+1, strlen(vers->name));
355 }
356}
357
358/*
359 * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC.
360 * seem to be defined (value set later).
361 */
362static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
363{
364 unsigned int i;
365
366 for (i = 1; i < numsyms; i++) {
367 if (syms[i].st_shndx == SHN_UNDEF) {
368 char *name = strtab + syms[i].st_name;
369 if (name[0] == '.') {
370 if (strcmp(name+1, "TOC.") == 0)
371 syms[i].st_shndx = SHN_ABS;
372 syms[i].st_name++;
373 }
374 }
375 }
376}
377
378static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs,
379 const char *strtab,
380 unsigned int symindex)
381{
382 unsigned int i, numsyms;
383 Elf64_Sym *syms;
384
385 syms = (Elf64_Sym *)sechdrs[symindex].sh_addr;
386 numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym);
387
388 for (i = 1; i < numsyms; i++) {
389 if (syms[i].st_shndx == SHN_ABS
390 && strcmp(strtab + syms[i].st_name, "TOC.") == 0)
391 return &syms[i];
392 }
393 return NULL;
394}
395#endif /* CONFIG_PPC_KERNEL_PCREL */
396
397bool module_init_section(const char *name)
398{
399 /* We don't handle .init for the moment: always return false. */
400 return false;
401}
402
403int module_frob_arch_sections(Elf64_Ehdr *hdr,
404 Elf64_Shdr *sechdrs,
405 char *secstrings,
406 struct module *me)
407{
408 unsigned int i;
409
410 /* Find .toc and .stubs sections, symtab and strtab */
411 for (i = 1; i < hdr->e_shnum; i++) {
412 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
413 me->arch.stubs_section = i;
414#ifdef CONFIG_PPC_KERNEL_PCREL
415 else if (strcmp(secstrings + sechdrs[i].sh_name, ".data..percpu") == 0)
416 me->arch.pcpu_section = i;
417 else if (strcmp(secstrings + sechdrs[i].sh_name, ".mygot") == 0) {
418 me->arch.got_section = i;
419 if (sechdrs[i].sh_addralign < 8)
420 sechdrs[i].sh_addralign = 8;
421 }
422#else
423 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) {
424 me->arch.toc_section = i;
425 if (sechdrs[i].sh_addralign < 8)
426 sechdrs[i].sh_addralign = 8;
427 }
428 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
429 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
430 sechdrs[i].sh_size);
431
432 if (sechdrs[i].sh_type == SHT_SYMTAB)
433 dedotify((void *)hdr + sechdrs[i].sh_offset,
434 sechdrs[i].sh_size / sizeof(Elf64_Sym),
435 (void *)hdr
436 + sechdrs[sechdrs[i].sh_link].sh_offset);
437#endif
438 }
439
440 if (!me->arch.stubs_section) {
441 pr_err("%s: doesn't contain .stubs.\n", me->name);
442 return -ENOEXEC;
443 }
444
445#ifdef CONFIG_PPC_KERNEL_PCREL
446 if (!me->arch.got_section) {
447 pr_err("%s: doesn't contain .mygot.\n", me->name);
448 return -ENOEXEC;
449 }
450
451 /* Override the got size */
452 sechdrs[me->arch.got_section].sh_size = get_got_size(hdr, sechdrs, me);
453#else
454 /* If we don't have a .toc, just use .stubs. We need to set r2
455 to some reasonable value in case the module calls out to
456 other functions via a stub, or if a function pointer escapes
457 the module by some means. */
458 if (!me->arch.toc_section)
459 me->arch.toc_section = me->arch.stubs_section;
460#endif
461
462 /* Override the stubs size */
463 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
464
465 return 0;
466}
467
468#if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)
469
470static u32 stub_insns[] = {
471#ifdef CONFIG_PPC_KERNEL_PCREL
472 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)),
473 PPC_RAW_NOP(), /* align the prefix insn */
474 /* paddi r12,r12,addr */
475 PPC_PREFIX_MLS | __PPC_PRFX_R(0),
476 PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12),
477 PPC_RAW_MTCTR(_R12),
478 PPC_RAW_BCTR(),
479#else
480 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
481 PPC_RAW_ADDIS(_R12, _R12, 0),
482 PPC_RAW_ADDI(_R12, _R12, 0),
483 PPC_RAW_MTCTR(_R12),
484 PPC_RAW_BCTR(),
485#endif
486};
487
488/*
489 * For mprofile-kernel we use a special stub for ftrace_caller() because we
490 * can't rely on r2 containing this module's TOC when we enter the stub.
491 *
492 * That can happen if the function calling us didn't need to use the toc. In
493 * that case it won't have setup r2, and the r2 value will be either the
494 * kernel's toc, or possibly another modules toc.
495 *
496 * To deal with that this stub uses the kernel toc, which is always accessible
497 * via the paca (in r13). The target (ftrace_caller()) is responsible for
498 * saving and restoring the toc before returning.
499 */
500static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
501 unsigned long addr,
502 struct module *me)
503{
504 long reladdr;
505
506 if ((unsigned long)entry->jump % 8 != 0) {
507 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
508 return 0;
509 }
510
511 BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump));
512 memcpy(entry->jump, stub_insns, sizeof(stub_insns));
513
514 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
515 /* Stub uses address relative to kernel base (from the paca) */
516 reladdr = addr - local_paca->kernelbase;
517 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
518 pr_err("%s: Address of %ps out of range of 34-bit relative address.\n",
519 me->name, (void *)addr);
520 return 0;
521 }
522
523 entry->jump[2] |= IMM_H18(reladdr);
524 entry->jump[3] |= IMM_L(reladdr);
525 } else {
526 /* Stub uses address relative to kernel toc (from the paca) */
527 reladdr = addr - kernel_toc_addr();
528 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
529 pr_err("%s: Address of %ps out of range of kernel_toc.\n",
530 me->name, (void *)addr);
531 return 0;
532 }
533
534 entry->jump[1] |= PPC_HA(reladdr);
535 entry->jump[2] |= PPC_LO(reladdr);
536 }
537
538 /* Even though we don't use funcdata in the stub, it's needed elsewhere. */
539 entry->funcdata = func_desc(addr);
540 entry->magic = STUB_MAGIC;
541
542 return 1;
543}
544
545static bool is_mprofile_ftrace_call(const char *name)
546{
547 if (!strcmp("_mcount", name))
548 return true;
549#ifdef CONFIG_DYNAMIC_FTRACE
550 if (!strcmp("ftrace_caller", name))
551 return true;
552#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
553 if (!strcmp("ftrace_regs_caller", name))
554 return true;
555#endif
556#endif
557
558 return false;
559}
560#else
561static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
562 unsigned long addr,
563 struct module *me)
564{
565 return 0;
566}
567
568static bool is_mprofile_ftrace_call(const char *name)
569{
570 return false;
571}
572#endif
573
574/*
575 * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
576 * value maximum span in an instruction which uses a signed offset). Round down
577 * to a 256 byte boundary for the odd case where we are setting up r2 without a
578 * .toc section.
579 */
580static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me)
581{
582#ifndef CONFIG_PPC_KERNEL_PCREL
583 return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
584#else
585 return -1;
586#endif
587}
588
589/* Patch stub to reference function and correct r2 value. */
590static inline int create_stub(const Elf64_Shdr *sechdrs,
591 struct ppc64_stub_entry *entry,
592 unsigned long addr,
593 struct module *me,
594 const char *name)
595{
596 long reladdr;
597 func_desc_t desc;
598 int i;
599
600 if (is_mprofile_ftrace_call(name))
601 return create_ftrace_stub(entry, addr, me);
602
603 if ((unsigned long)entry->jump % 8 != 0) {
604 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
605 return 0;
606 }
607
608 BUILD_BUG_ON(sizeof(ppc64_stub_insns) > sizeof(entry->jump));
609 for (i = 0; i < ARRAY_SIZE(ppc64_stub_insns); i++) {
610 if (patch_instruction(&entry->jump[i],
611 ppc_inst(ppc64_stub_insns[i])))
612 return 0;
613 }
614
615 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
616 /* Stub uses address relative to itself! */
617 reladdr = 0 + offsetof(struct ppc64_stub_entry, funcdata);
618 BUILD_BUG_ON(reladdr != 32);
619 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
620 pr_err("%s: Address of %p out of range of 34-bit relative address.\n",
621 me->name, (void *)reladdr);
622 return 0;
623 }
624 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
625
626 /* May not even need this if we're relative to 0 */
627 if (patch_instruction(&entry->jump[0],
628 ppc_inst_prefix(entry->jump[0] | IMM_H18(reladdr),
629 entry->jump[1] | IMM_L(reladdr))))
630 return 0;
631
632 } else {
633 /* Stub uses address relative to r2. */
634 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
635 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
636 pr_err("%s: Address %p of stub out of range of %p.\n",
637 me->name, (void *)reladdr, (void *)my_r2);
638 return 0;
639 }
640 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
641
642 if (patch_instruction(&entry->jump[0],
643 ppc_inst(entry->jump[0] | PPC_HA(reladdr))))
644 return 0;
645
646 if (patch_instruction(&entry->jump[1],
647 ppc_inst(entry->jump[1] | PPC_LO(reladdr))))
648 return 0;
649 }
650
651 // func_desc_t is 8 bytes if ABIv2, else 16 bytes
652 desc = func_desc(addr);
653 for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) {
654 if (patch_instruction(((u32 *)&entry->funcdata) + i,
655 ppc_inst(((u32 *)(&desc))[i])))
656 return 0;
657 }
658
659 if (patch_instruction(&entry->magic, ppc_inst(STUB_MAGIC)))
660 return 0;
661
662 return 1;
663}
664
665/* Create stub to jump to function described in this OPD/ptr: we need the
666 stub to set up the TOC ptr (r2) for the function. */
667static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
668 unsigned long addr,
669 struct module *me,
670 const char *name)
671{
672 struct ppc64_stub_entry *stubs;
673 unsigned int i, num_stubs;
674
675 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
676
677 /* Find this stub, or if that fails, the next avail. entry */
678 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
679 for (i = 0; stub_func_addr(stubs[i].funcdata); i++) {
680 if (WARN_ON(i >= num_stubs))
681 return 0;
682
683 if (stub_func_addr(stubs[i].funcdata) == func_addr(addr))
684 return (unsigned long)&stubs[i];
685 }
686
687 if (!create_stub(sechdrs, &stubs[i], addr, me, name))
688 return 0;
689
690 return (unsigned long)&stubs[i];
691}
692
693#ifdef CONFIG_PPC_KERNEL_PCREL
694/* Create GOT to load the location described in this ptr */
695static unsigned long got_for_addr(const Elf64_Shdr *sechdrs,
696 unsigned long addr,
697 struct module *me,
698 const char *name)
699{
700 struct ppc64_got_entry *got;
701 unsigned int i, num_got;
702
703 if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
704 return addr;
705
706 num_got = sechdrs[me->arch.got_section].sh_size / sizeof(*got);
707
708 /* Find this stub, or if that fails, the next avail. entry */
709 got = (void *)sechdrs[me->arch.got_section].sh_addr;
710 for (i = 0; got[i].addr; i++) {
711 if (WARN_ON(i >= num_got))
712 return 0;
713
714 if (got[i].addr == addr)
715 return (unsigned long)&got[i];
716 }
717
718 got[i].addr = addr;
719
720 return (unsigned long)&got[i];
721}
722#endif
723
724/* We expect a noop next: if it is, replace it with instruction to
725 restore r2. */
726static int restore_r2(const char *name, u32 *instruction, struct module *me)
727{
728 u32 *prev_insn = instruction - 1;
729 u32 insn_val = *instruction;
730
731 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
732 return 0;
733
734 if (is_mprofile_ftrace_call(name))
735 return 0;
736
737 /*
738 * Make sure the branch isn't a sibling call. Sibling calls aren't
739 * "link" branches and they don't return, so they don't need the r2
740 * restore afterwards.
741 */
742 if (!instr_is_relative_link_branch(ppc_inst(*prev_insn)))
743 return 0;
744
745 /*
746 * For livepatch, the restore r2 instruction might have already been
747 * written previously, if the referenced symbol is in a previously
748 * unloaded module which is now being loaded again. In that case, skip
749 * the warning and the instruction write.
750 */
751 if (insn_val == PPC_INST_LD_TOC)
752 return 0;
753
754 if (insn_val != PPC_RAW_NOP()) {
755 pr_err("%s: Expected nop after call, got %08x at %pS\n",
756 me->name, insn_val, instruction);
757 return -ENOEXEC;
758 }
759
760 /* ld r2,R2_STACK_OFFSET(r1) */
761 return patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC));
762}
763
764int apply_relocate_add(Elf64_Shdr *sechdrs,
765 const char *strtab,
766 unsigned int symindex,
767 unsigned int relsec,
768 struct module *me)
769{
770 unsigned int i;
771 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
772 Elf64_Sym *sym;
773 unsigned long *location;
774 unsigned long value;
775
776 pr_debug("Applying ADD relocate section %u to %u\n", relsec,
777 sechdrs[relsec].sh_info);
778
779#ifndef CONFIG_PPC_KERNEL_PCREL
780 /* First time we're called, we can fix up .TOC. */
781 if (!me->arch.toc_fixed) {
782 sym = find_dot_toc(sechdrs, strtab, symindex);
783 /* It's theoretically possible that a module doesn't want a
784 * .TOC. so don't fail it just for that. */
785 if (sym)
786 sym->st_value = my_r2(sechdrs, me);
787 me->arch.toc_fixed = true;
788 }
789#endif
790 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
791 /* This is where to make the change */
792 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
793 + rela[i].r_offset;
794 /* This is the symbol it is referring to */
795 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
796 + ELF64_R_SYM(rela[i].r_info);
797
798 pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n",
799 location, (long)ELF64_R_TYPE(rela[i].r_info),
800 strtab + sym->st_name, (unsigned long)sym->st_value,
801 (long)rela[i].r_addend);
802
803 /* `Everything is relative'. */
804 value = sym->st_value + rela[i].r_addend;
805
806 switch (ELF64_R_TYPE(rela[i].r_info)) {
807 case R_PPC64_ADDR32:
808 /* Simply set it */
809 *(u32 *)location = value;
810 break;
811
812 case R_PPC64_ADDR64:
813 /* Simply set it */
814 *(unsigned long *)location = value;
815 break;
816
817#ifndef CONFIG_PPC_KERNEL_PCREL
818 case R_PPC64_TOC:
819 *(unsigned long *)location = my_r2(sechdrs, me);
820 break;
821
822 case R_PPC64_TOC16:
823 /* Subtract TOC pointer */
824 value -= my_r2(sechdrs, me);
825 if (value + 0x8000 > 0xffff) {
826 pr_err("%s: bad TOC16 relocation (0x%lx)\n",
827 me->name, value);
828 return -ENOEXEC;
829 }
830 *((uint16_t *) location)
831 = (*((uint16_t *) location) & ~0xffff)
832 | (value & 0xffff);
833 break;
834
835 case R_PPC64_TOC16_LO:
836 /* Subtract TOC pointer */
837 value -= my_r2(sechdrs, me);
838 *((uint16_t *) location)
839 = (*((uint16_t *) location) & ~0xffff)
840 | (value & 0xffff);
841 break;
842
843 case R_PPC64_TOC16_DS:
844 /* Subtract TOC pointer */
845 value -= my_r2(sechdrs, me);
846 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
847 pr_err("%s: bad TOC16_DS relocation (0x%lx)\n",
848 me->name, value);
849 return -ENOEXEC;
850 }
851 *((uint16_t *) location)
852 = (*((uint16_t *) location) & ~0xfffc)
853 | (value & 0xfffc);
854 break;
855
856 case R_PPC64_TOC16_LO_DS:
857 /* Subtract TOC pointer */
858 value -= my_r2(sechdrs, me);
859 if ((value & 3) != 0) {
860 pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n",
861 me->name, value);
862 return -ENOEXEC;
863 }
864 *((uint16_t *) location)
865 = (*((uint16_t *) location) & ~0xfffc)
866 | (value & 0xfffc);
867 break;
868
869 case R_PPC64_TOC16_HA:
870 /* Subtract TOC pointer */
871 value -= my_r2(sechdrs, me);
872 value = ((value + 0x8000) >> 16);
873 *((uint16_t *) location)
874 = (*((uint16_t *) location) & ~0xffff)
875 | (value & 0xffff);
876 break;
877#endif
878
879 case R_PPC_REL24:
880#ifdef CONFIG_PPC_KERNEL_PCREL
881 /* PCREL still generates REL24 for mcount */
882 case R_PPC64_REL24_NOTOC:
883#endif
884 /* FIXME: Handle weak symbols here --RR */
885 if (sym->st_shndx == SHN_UNDEF ||
886 sym->st_shndx == SHN_LIVEPATCH) {
887 /* External: go via stub */
888 value = stub_for_addr(sechdrs, value, me,
889 strtab + sym->st_name);
890 if (!value)
891 return -ENOENT;
892 if (restore_r2(strtab + sym->st_name,
893 (u32 *)location + 1, me))
894 return -ENOEXEC;
895 } else
896 value += local_entry_offset(sym);
897
898 /* Convert value to relative */
899 value -= (unsigned long)location;
900 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
901 pr_err("%s: REL24 %li out of range!\n",
902 me->name, (long int)value);
903 return -ENOEXEC;
904 }
905
906 /* Only replace bits 2 through 26 */
907 value = (*(uint32_t *)location & ~PPC_LI_MASK) | PPC_LI(value);
908
909 if (patch_instruction((u32 *)location, ppc_inst(value)))
910 return -EFAULT;
911
912 break;
913
914 case R_PPC64_REL64:
915 /* 64 bits relative (used by features fixups) */
916 *location = value - (unsigned long)location;
917 break;
918
919 case R_PPC64_REL32:
920 /* 32 bits relative (used by relative exception tables) */
921 /* Convert value to relative */
922 value -= (unsigned long)location;
923 if (value + 0x80000000 > 0xffffffff) {
924 pr_err("%s: REL32 %li out of range!\n",
925 me->name, (long int)value);
926 return -ENOEXEC;
927 }
928 *(u32 *)location = value;
929 break;
930
931#ifdef CONFIG_PPC_KERNEL_PCREL
932 case R_PPC64_PCREL34: {
933 unsigned long absvalue = value;
934
935 /* Convert value to relative */
936 value -= (unsigned long)location;
937
938 if (value + 0x200000000 > 0x3ffffffff) {
939 if (sym->st_shndx != me->arch.pcpu_section) {
940 pr_err("%s: REL34 %li out of range!\n",
941 me->name, (long)value);
942 return -ENOEXEC;
943 }
944
945 /*
946 * per-cpu section is special cased because
947 * it is moved during loading, so has to be
948 * converted to use GOT.
949 */
950 value = got_for_addr(sechdrs, absvalue, me,
951 strtab + sym->st_name);
952 if (!value)
953 return -ENOENT;
954 value -= (unsigned long)location;
955
956 /* Turn pla into pld */
957 if (patch_instruction((u32 *)location,
958 ppc_inst_prefix((*(u32 *)location & ~0x02000000),
959 (*((u32 *)location + 1) & ~0xf8000000) | 0xe4000000)))
960 return -EFAULT;
961 }
962
963 if (patch_instruction((u32 *)location,
964 ppc_inst_prefix((*(u32 *)location & ~0x3ffff) | IMM_H18(value),
965 (*((u32 *)location + 1) & ~0xffff) | IMM_L(value))))
966 return -EFAULT;
967
968 break;
969 }
970
971#else
972 case R_PPC64_TOCSAVE:
973 /*
974 * Marker reloc indicates we don't have to save r2.
975 * That would only save us one instruction, so ignore
976 * it.
977 */
978 break;
979#endif
980
981 case R_PPC64_ENTRY:
982 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
983 break;
984
985 /*
986 * Optimize ELFv2 large code model entry point if
987 * the TOC is within 2GB range of current location.
988 */
989 value = my_r2(sechdrs, me) - (unsigned long)location;
990 if (value + 0x80008000 > 0xffffffff)
991 break;
992 /*
993 * Check for the large code model prolog sequence:
994 * ld r2, ...(r12)
995 * add r2, r2, r12
996 */
997 if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0))
998 break;
999 if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12))
1000 break;
1001 /*
1002 * If found, replace it with:
1003 * addis r2, r12, (.TOC.-func)@ha
1004 * addi r2, r2, (.TOC.-func)@l
1005 */
1006 ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value));
1007 ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value));
1008 break;
1009
1010 case R_PPC64_REL16_HA:
1011 /* Subtract location pointer */
1012 value -= (unsigned long)location;
1013 value = ((value + 0x8000) >> 16);
1014 *((uint16_t *) location)
1015 = (*((uint16_t *) location) & ~0xffff)
1016 | (value & 0xffff);
1017 break;
1018
1019 case R_PPC64_REL16_LO:
1020 /* Subtract location pointer */
1021 value -= (unsigned long)location;
1022 *((uint16_t *) location)
1023 = (*((uint16_t *) location) & ~0xffff)
1024 | (value & 0xffff);
1025 break;
1026
1027#ifdef CONFIG_PPC_KERNEL_PCREL
1028 case R_PPC64_GOT_PCREL34:
1029 value = got_for_addr(sechdrs, value, me,
1030 strtab + sym->st_name);
1031 if (!value)
1032 return -ENOENT;
1033 value -= (unsigned long)location;
1034 ((uint32_t *)location)[0] = (((uint32_t *)location)[0] & ~0x3ffff) |
1035 ((value >> 16) & 0x3ffff);
1036 ((uint32_t *)location)[1] = (((uint32_t *)location)[1] & ~0xffff) |
1037 (value & 0xffff);
1038 break;
1039#endif
1040
1041 default:
1042 pr_err("%s: Unknown ADD relocation: %lu\n",
1043 me->name,
1044 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
1045 return -ENOEXEC;
1046 }
1047 }
1048
1049 return 0;
1050}
1051
1052#ifdef CONFIG_DYNAMIC_FTRACE
1053int module_trampoline_target(struct module *mod, unsigned long addr,
1054 unsigned long *target)
1055{
1056 struct ppc64_stub_entry *stub;
1057 func_desc_t funcdata;
1058 u32 magic;
1059
1060 if (!within_module_core(addr, mod)) {
1061 pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
1062 return -EFAULT;
1063 }
1064
1065 stub = (struct ppc64_stub_entry *)addr;
1066
1067 if (copy_from_kernel_nofault(&magic, &stub->magic,
1068 sizeof(magic))) {
1069 pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
1070 return -EFAULT;
1071 }
1072
1073 if (magic != STUB_MAGIC) {
1074 pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
1075 return -EFAULT;
1076 }
1077
1078 if (copy_from_kernel_nofault(&funcdata, &stub->funcdata,
1079 sizeof(funcdata))) {
1080 pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
1081 return -EFAULT;
1082 }
1083
1084 *target = stub_func_addr(funcdata);
1085
1086 return 0;
1087}
1088
1089int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
1090{
1091 mod->arch.tramp = stub_for_addr(sechdrs,
1092 (unsigned long)ftrace_caller,
1093 mod,
1094 "ftrace_caller");
1095#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1096 mod->arch.tramp_regs = stub_for_addr(sechdrs,
1097 (unsigned long)ftrace_regs_caller,
1098 mod,
1099 "ftrace_regs_caller");
1100 if (!mod->arch.tramp_regs)
1101 return -ENOENT;
1102#endif
1103
1104 if (!mod->arch.tramp)
1105 return -ENOENT;
1106
1107 return 0;
1108}
1109#endif
1/* Kernel module help for PPC64.
2 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17*/
18#include <linux/module.h>
19#include <linux/elf.h>
20#include <linux/moduleloader.h>
21#include <linux/err.h>
22#include <linux/vmalloc.h>
23#include <linux/ftrace.h>
24#include <linux/bug.h>
25#include <asm/module.h>
26#include <asm/firmware.h>
27#include <asm/code-patching.h>
28#include <linux/sort.h>
29#include <asm/setup.h>
30
31/* FIXME: We don't do .init separately. To do this, we'd need to have
32 a separate r2 value in the init and core section, and stub between
33 them, too.
34
35 Using a magic allocator which places modules within 32MB solves
36 this, and makes other things simpler. Anton?
37 --RR. */
38#if 0
39#define DEBUGP printk
40#else
41#define DEBUGP(fmt , ...)
42#endif
43
44/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
45 the kernel itself). But on PPC64, these need to be used for every
46 jump, actually, to reset r2 (TOC+0x8000). */
47struct ppc64_stub_entry
48{
49 /* 28 byte jump instruction sequence (7 instructions) */
50 unsigned char jump[28];
51 unsigned char unused[4];
52 /* Data for the above code */
53 struct ppc64_opd_entry opd;
54};
55
56/* We use a stub to fix up r2 (TOC ptr) and to jump to the (external)
57 function which may be more than 24-bits away. We could simply
58 patch the new r2 value and function pointer into the stub, but it's
59 significantly shorter to put these values at the end of the stub
60 code, and patch the stub address (32-bits relative to the TOC ptr,
61 r2) into the stub. */
62static struct ppc64_stub_entry ppc64_stub =
63{ .jump = {
64#ifdef __LITTLE_ENDIAN__
65 0x00, 0x00, 0x82, 0x3d, /* addis r12,r2, <high> */
66 0x00, 0x00, 0x8c, 0x39, /* addi r12,r12, <low> */
67 /* Save current r2 value in magic place on the stack. */
68 0x28, 0x00, 0x41, 0xf8, /* std r2,40(r1) */
69 0x20, 0x00, 0x6c, 0xe9, /* ld r11,32(r12) */
70 0x28, 0x00, 0x4c, 0xe8, /* ld r2,40(r12) */
71 0xa6, 0x03, 0x69, 0x7d, /* mtctr r11 */
72 0x20, 0x04, 0x80, 0x4e /* bctr */
73#else
74 0x3d, 0x82, 0x00, 0x00, /* addis r12,r2, <high> */
75 0x39, 0x8c, 0x00, 0x00, /* addi r12,r12, <low> */
76 /* Save current r2 value in magic place on the stack. */
77 0xf8, 0x41, 0x00, 0x28, /* std r2,40(r1) */
78 0xe9, 0x6c, 0x00, 0x20, /* ld r11,32(r12) */
79 0xe8, 0x4c, 0x00, 0x28, /* ld r2,40(r12) */
80 0x7d, 0x69, 0x03, 0xa6, /* mtctr r11 */
81 0x4e, 0x80, 0x04, 0x20 /* bctr */
82#endif
83} };
84
85/* Count how many different 24-bit relocations (different symbol,
86 different addend) */
87static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num)
88{
89 unsigned int i, r_info, r_addend, _count_relocs;
90
91 /* FIXME: Only count external ones --RR */
92 _count_relocs = 0;
93 r_info = 0;
94 r_addend = 0;
95 for (i = 0; i < num; i++)
96 /* Only count 24-bit relocs, others don't need stubs */
97 if (ELF64_R_TYPE(rela[i].r_info) == R_PPC_REL24 &&
98 (r_info != ELF64_R_SYM(rela[i].r_info) ||
99 r_addend != rela[i].r_addend)) {
100 _count_relocs++;
101 r_info = ELF64_R_SYM(rela[i].r_info);
102 r_addend = rela[i].r_addend;
103 }
104
105 return _count_relocs;
106}
107
108static int relacmp(const void *_x, const void *_y)
109{
110 const Elf64_Rela *x, *y;
111
112 y = (Elf64_Rela *)_x;
113 x = (Elf64_Rela *)_y;
114
115 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
116 * make the comparison cheaper/faster. It won't affect the sorting or
117 * the counting algorithms' performance
118 */
119 if (x->r_info < y->r_info)
120 return -1;
121 else if (x->r_info > y->r_info)
122 return 1;
123 else if (x->r_addend < y->r_addend)
124 return -1;
125 else if (x->r_addend > y->r_addend)
126 return 1;
127 else
128 return 0;
129}
130
131static void relaswap(void *_x, void *_y, int size)
132{
133 uint64_t *x, *y, tmp;
134 int i;
135
136 y = (uint64_t *)_x;
137 x = (uint64_t *)_y;
138
139 for (i = 0; i < sizeof(Elf64_Rela) / sizeof(uint64_t); i++) {
140 tmp = x[i];
141 x[i] = y[i];
142 y[i] = tmp;
143 }
144}
145
146/* Get size of potential trampolines required. */
147static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
148 const Elf64_Shdr *sechdrs)
149{
150 /* One extra reloc so it's always 0-funcaddr terminated */
151 unsigned long relocs = 1;
152 unsigned i;
153
154 /* Every relocated section... */
155 for (i = 1; i < hdr->e_shnum; i++) {
156 if (sechdrs[i].sh_type == SHT_RELA) {
157 DEBUGP("Found relocations in section %u\n", i);
158 DEBUGP("Ptr: %p. Number: %lu\n",
159 (void *)sechdrs[i].sh_addr,
160 sechdrs[i].sh_size / sizeof(Elf64_Rela));
161
162 /* Sort the relocation information based on a symbol and
163 * addend key. This is a stable O(n*log n) complexity
164 * alogrithm but it will reduce the complexity of
165 * count_relocs() to linear complexity O(n)
166 */
167 sort((void *)sechdrs[i].sh_addr,
168 sechdrs[i].sh_size / sizeof(Elf64_Rela),
169 sizeof(Elf64_Rela), relacmp, relaswap);
170
171 relocs += count_relocs((void *)sechdrs[i].sh_addr,
172 sechdrs[i].sh_size
173 / sizeof(Elf64_Rela));
174 }
175 }
176
177#ifdef CONFIG_DYNAMIC_FTRACE
178 /* make the trampoline to the ftrace_caller */
179 relocs++;
180#endif
181
182 DEBUGP("Looks like a total of %lu stubs, max\n", relocs);
183 return relocs * sizeof(struct ppc64_stub_entry);
184}
185
186static void dedotify_versions(struct modversion_info *vers,
187 unsigned long size)
188{
189 struct modversion_info *end;
190
191 for (end = (void *)vers + size; vers < end; vers++)
192 if (vers->name[0] == '.')
193 memmove(vers->name, vers->name+1, strlen(vers->name));
194}
195
196/* Undefined symbols which refer to .funcname, hack to funcname */
197static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
198{
199 unsigned int i;
200
201 for (i = 1; i < numsyms; i++) {
202 if (syms[i].st_shndx == SHN_UNDEF) {
203 char *name = strtab + syms[i].st_name;
204 if (name[0] == '.')
205 memmove(name, name+1, strlen(name));
206 }
207 }
208}
209
210int module_frob_arch_sections(Elf64_Ehdr *hdr,
211 Elf64_Shdr *sechdrs,
212 char *secstrings,
213 struct module *me)
214{
215 unsigned int i;
216
217 /* Find .toc and .stubs sections, symtab and strtab */
218 for (i = 1; i < hdr->e_shnum; i++) {
219 char *p;
220 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
221 me->arch.stubs_section = i;
222 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0)
223 me->arch.toc_section = i;
224 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
225 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
226 sechdrs[i].sh_size);
227
228 /* We don't handle .init for the moment: rename to _init */
229 while ((p = strstr(secstrings + sechdrs[i].sh_name, ".init")))
230 p[0] = '_';
231
232 if (sechdrs[i].sh_type == SHT_SYMTAB)
233 dedotify((void *)hdr + sechdrs[i].sh_offset,
234 sechdrs[i].sh_size / sizeof(Elf64_Sym),
235 (void *)hdr
236 + sechdrs[sechdrs[i].sh_link].sh_offset);
237 }
238
239 if (!me->arch.stubs_section) {
240 printk("%s: doesn't contain .stubs.\n", me->name);
241 return -ENOEXEC;
242 }
243
244 /* If we don't have a .toc, just use .stubs. We need to set r2
245 to some reasonable value in case the module calls out to
246 other functions via a stub, or if a function pointer escapes
247 the module by some means. */
248 if (!me->arch.toc_section)
249 me->arch.toc_section = me->arch.stubs_section;
250
251 /* Override the stubs size */
252 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
253 return 0;
254}
255
256/* r2 is the TOC pointer: it actually points 0x8000 into the TOC (this
257 gives the value maximum span in an instruction which uses a signed
258 offset) */
259static inline unsigned long my_r2(Elf64_Shdr *sechdrs, struct module *me)
260{
261 return sechdrs[me->arch.toc_section].sh_addr + 0x8000;
262}
263
264/* Both low and high 16 bits are added as SIGNED additions, so if low
265 16 bits has high bit set, high 16 bits must be adjusted. These
266 macros do that (stolen from binutils). */
267#define PPC_LO(v) ((v) & 0xffff)
268#define PPC_HI(v) (((v) >> 16) & 0xffff)
269#define PPC_HA(v) PPC_HI ((v) + 0x8000)
270
271/* Patch stub to reference function and correct r2 value. */
272static inline int create_stub(Elf64_Shdr *sechdrs,
273 struct ppc64_stub_entry *entry,
274 struct ppc64_opd_entry *opd,
275 struct module *me)
276{
277 Elf64_Half *loc1, *loc2;
278 long reladdr;
279
280 *entry = ppc64_stub;
281
282#ifdef __LITTLE_ENDIAN__
283 loc1 = (Elf64_Half *)&entry->jump[0];
284 loc2 = (Elf64_Half *)&entry->jump[4];
285#else
286 loc1 = (Elf64_Half *)&entry->jump[2];
287 loc2 = (Elf64_Half *)&entry->jump[6];
288#endif
289
290 /* Stub uses address relative to r2. */
291 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
292 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
293 printk("%s: Address %p of stub out of range of %p.\n",
294 me->name, (void *)reladdr, (void *)my_r2);
295 return 0;
296 }
297 DEBUGP("Stub %p get data from reladdr %li\n", entry, reladdr);
298
299 *loc1 = PPC_HA(reladdr);
300 *loc2 = PPC_LO(reladdr);
301 entry->opd.funcaddr = opd->funcaddr;
302 entry->opd.r2 = opd->r2;
303 return 1;
304}
305
306/* Create stub to jump to function described in this OPD: we need the
307 stub to set up the TOC ptr (r2) for the function. */
308static unsigned long stub_for_addr(Elf64_Shdr *sechdrs,
309 unsigned long opdaddr,
310 struct module *me)
311{
312 struct ppc64_stub_entry *stubs;
313 struct ppc64_opd_entry *opd = (void *)opdaddr;
314 unsigned int i, num_stubs;
315
316 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
317
318 /* Find this stub, or if that fails, the next avail. entry */
319 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
320 for (i = 0; stubs[i].opd.funcaddr; i++) {
321 BUG_ON(i >= num_stubs);
322
323 if (stubs[i].opd.funcaddr == opd->funcaddr)
324 return (unsigned long)&stubs[i];
325 }
326
327 if (!create_stub(sechdrs, &stubs[i], opd, me))
328 return 0;
329
330 return (unsigned long)&stubs[i];
331}
332
333/* We expect a noop next: if it is, replace it with instruction to
334 restore r2. */
335static int restore_r2(u32 *instruction, struct module *me)
336{
337 if (*instruction != PPC_INST_NOP) {
338 printk("%s: Expect noop after relocate, got %08x\n",
339 me->name, *instruction);
340 return 0;
341 }
342 *instruction = 0xe8410028; /* ld r2,40(r1) */
343 return 1;
344}
345
346int apply_relocate_add(Elf64_Shdr *sechdrs,
347 const char *strtab,
348 unsigned int symindex,
349 unsigned int relsec,
350 struct module *me)
351{
352 unsigned int i;
353 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
354 Elf64_Sym *sym;
355 unsigned long *location;
356 unsigned long value;
357
358 DEBUGP("Applying ADD relocate section %u to %u\n", relsec,
359 sechdrs[relsec].sh_info);
360 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
361 /* This is where to make the change */
362 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
363 + rela[i].r_offset;
364 /* This is the symbol it is referring to */
365 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
366 + ELF64_R_SYM(rela[i].r_info);
367
368 DEBUGP("RELOC at %p: %li-type as %s (%lu) + %li\n",
369 location, (long)ELF64_R_TYPE(rela[i].r_info),
370 strtab + sym->st_name, (unsigned long)sym->st_value,
371 (long)rela[i].r_addend);
372
373 /* `Everything is relative'. */
374 value = sym->st_value + rela[i].r_addend;
375
376 switch (ELF64_R_TYPE(rela[i].r_info)) {
377 case R_PPC64_ADDR32:
378 /* Simply set it */
379 *(u32 *)location = value;
380 break;
381
382 case R_PPC64_ADDR64:
383 /* Simply set it */
384 *(unsigned long *)location = value;
385 break;
386
387 case R_PPC64_TOC:
388 *(unsigned long *)location = my_r2(sechdrs, me);
389 break;
390
391 case R_PPC64_TOC16:
392 /* Subtract TOC pointer */
393 value -= my_r2(sechdrs, me);
394 if (value + 0x8000 > 0xffff) {
395 printk("%s: bad TOC16 relocation (%lu)\n",
396 me->name, value);
397 return -ENOEXEC;
398 }
399 *((uint16_t *) location)
400 = (*((uint16_t *) location) & ~0xffff)
401 | (value & 0xffff);
402 break;
403
404 case R_PPC64_TOC16_LO:
405 /* Subtract TOC pointer */
406 value -= my_r2(sechdrs, me);
407 *((uint16_t *) location)
408 = (*((uint16_t *) location) & ~0xffff)
409 | (value & 0xffff);
410 break;
411
412 case R_PPC64_TOC16_DS:
413 /* Subtract TOC pointer */
414 value -= my_r2(sechdrs, me);
415 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
416 printk("%s: bad TOC16_DS relocation (%lu)\n",
417 me->name, value);
418 return -ENOEXEC;
419 }
420 *((uint16_t *) location)
421 = (*((uint16_t *) location) & ~0xfffc)
422 | (value & 0xfffc);
423 break;
424
425 case R_PPC64_TOC16_LO_DS:
426 /* Subtract TOC pointer */
427 value -= my_r2(sechdrs, me);
428 if ((value & 3) != 0) {
429 printk("%s: bad TOC16_LO_DS relocation (%lu)\n",
430 me->name, value);
431 return -ENOEXEC;
432 }
433 *((uint16_t *) location)
434 = (*((uint16_t *) location) & ~0xfffc)
435 | (value & 0xfffc);
436 break;
437
438 case R_PPC64_TOC16_HA:
439 /* Subtract TOC pointer */
440 value -= my_r2(sechdrs, me);
441 value = ((value + 0x8000) >> 16);
442 *((uint16_t *) location)
443 = (*((uint16_t *) location) & ~0xffff)
444 | (value & 0xffff);
445 break;
446
447 case R_PPC_REL24:
448 /* FIXME: Handle weak symbols here --RR */
449 if (sym->st_shndx == SHN_UNDEF) {
450 /* External: go via stub */
451 value = stub_for_addr(sechdrs, value, me);
452 if (!value)
453 return -ENOENT;
454 if (!restore_r2((u32 *)location + 1, me))
455 return -ENOEXEC;
456 }
457
458 /* Convert value to relative */
459 value -= (unsigned long)location;
460 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
461 printk("%s: REL24 %li out of range!\n",
462 me->name, (long int)value);
463 return -ENOEXEC;
464 }
465
466 /* Only replace bits 2 through 26 */
467 *(uint32_t *)location
468 = (*(uint32_t *)location & ~0x03fffffc)
469 | (value & 0x03fffffc);
470 break;
471
472 case R_PPC64_REL64:
473 /* 64 bits relative (used by features fixups) */
474 *location = value - (unsigned long)location;
475 break;
476
477 default:
478 printk("%s: Unknown ADD relocation: %lu\n",
479 me->name,
480 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
481 return -ENOEXEC;
482 }
483 }
484
485#ifdef CONFIG_DYNAMIC_FTRACE
486 me->arch.toc = my_r2(sechdrs, me);
487 me->arch.tramp = stub_for_addr(sechdrs,
488 (unsigned long)ftrace_caller,
489 me);
490#endif
491
492 return 0;
493}