Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Kernel module help for PPC64.
3 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
4
5*/
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/module.h>
10#include <linux/elf.h>
11#include <linux/moduleloader.h>
12#include <linux/err.h>
13#include <linux/vmalloc.h>
14#include <linux/ftrace.h>
15#include <linux/bug.h>
16#include <linux/uaccess.h>
17#include <linux/kernel.h>
18#include <asm/module.h>
19#include <asm/firmware.h>
20#include <asm/text-patching.h>
21#include <linux/sort.h>
22#include <asm/setup.h>
23#include <asm/sections.h>
24#include <asm/inst.h>
25
26/* FIXME: We don't do .init separately. To do this, we'd need to have
27 a separate r2 value in the init and core section, and stub between
28 them, too.
29
30 Using a magic allocator which places modules within 32MB solves
31 this, and makes other things simpler. Anton?
32 --RR. */
33
34bool module_elf_check_arch(Elf_Ehdr *hdr)
35{
36 unsigned long abi_level = hdr->e_flags & 0x3;
37
38 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
39 return abi_level == 2;
40 else
41 return abi_level < 2;
42}
43
44#ifdef CONFIG_PPC64_ELF_ABI_V2
45
46static func_desc_t func_desc(unsigned long addr)
47{
48 func_desc_t desc = {
49 .addr = addr,
50 };
51
52 return desc;
53}
54
55/* PowerPC64 specific values for the Elf64_Sym st_other field. */
56#define STO_PPC64_LOCAL_BIT 5
57#define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT)
58#define PPC64_LOCAL_ENTRY_OFFSET(other) \
59 (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)
60
61static unsigned int local_entry_offset(const Elf64_Sym *sym)
62{
63 /* sym->st_other indicates offset to local entry point
64 * (otherwise it will assume r12 is the address of the start
65 * of function and try to derive r2 from it). */
66 return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other);
67}
68#else
69
70static func_desc_t func_desc(unsigned long addr)
71{
72 return *(struct func_desc *)addr;
73}
74static unsigned int local_entry_offset(const Elf64_Sym *sym)
75{
76 return 0;
77}
78
79void *dereference_module_function_descriptor(struct module *mod, void *ptr)
80{
81 if (ptr < (void *)mod->arch.start_opd ||
82 ptr >= (void *)mod->arch.end_opd)
83 return ptr;
84
85 return dereference_function_descriptor(ptr);
86}
87#endif
88
89static unsigned long func_addr(unsigned long addr)
90{
91 return func_desc(addr).addr;
92}
93
94static unsigned long stub_func_addr(func_desc_t func)
95{
96 return func.addr;
97}
98
99#define STUB_MAGIC 0x73747562 /* stub */
100
101/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
102 the kernel itself). But on PPC64, these need to be used for every
103 jump, actually, to reset r2 (TOC+0x8000). */
104struct ppc64_stub_entry {
105 /*
106 * 28 byte jump instruction sequence (7 instructions) that can
107 * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned
108 * with PCREL kernels that use prefix instructions in the stub.
109 */
110 u32 jump[7];
111 /* Used by ftrace to identify stubs */
112 u32 magic;
113 /* Data for the above code */
114 func_desc_t funcdata;
115} __aligned(8);
116
117struct ppc64_got_entry {
118 u64 addr;
119};
120
121/*
122 * PPC64 uses 24 bit jumps, but we need to jump into other modules or
123 * the kernel which may be further. So we jump to a stub.
124 *
125 * Target address and TOC are loaded from function descriptor in the
126 * ppc64_stub_entry.
127 *
128 * r12 is used to generate the target address, which is required for the
129 * ELFv2 global entry point calling convention.
130 *
131 * TOC handling:
132 * - PCREL does not have a TOC.
133 * - ELFv2 non-PCREL just has to save r2, the callee is responsible for
134 * setting its own TOC pointer at the global entry address.
135 * - ELFv1 must load the new TOC pointer from the function descriptor.
136 */
137static u32 ppc64_stub_insns[] = {
138#ifdef CONFIG_PPC_KERNEL_PCREL
139 /* pld r12,addr */
140 PPC_PREFIX_8LS | __PPC_PRFX_R(1),
141 PPC_INST_PLD | ___PPC_RT(_R12),
142#else
143 PPC_RAW_ADDIS(_R11, _R2, 0),
144 PPC_RAW_ADDI(_R11, _R11, 0),
145 /* Save current r2 value in magic place on the stack. */
146 PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET),
147 PPC_RAW_LD(_R12, _R11, 32),
148#ifdef CONFIG_PPC64_ELF_ABI_V1
149 /* Set up new r2 from function descriptor */
150 PPC_RAW_LD(_R2, _R11, 40),
151#endif
152#endif
153 PPC_RAW_MTCTR(_R12),
154 PPC_RAW_BCTR(),
155};
156
157/*
158 * Count how many different r_type relocations (different symbol,
159 * different addend).
160 */
161static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num,
162 unsigned long r_type)
163{
164 unsigned int i, r_info, r_addend, _count_relocs;
165
166 /* FIXME: Only count external ones --RR */
167 _count_relocs = 0;
168 r_info = 0;
169 r_addend = 0;
170 for (i = 0; i < num; i++)
171 /* Only count r_type relocs, others don't need stubs */
172 if (ELF64_R_TYPE(rela[i].r_info) == r_type &&
173 (r_info != ELF64_R_SYM(rela[i].r_info) ||
174 r_addend != rela[i].r_addend)) {
175 _count_relocs++;
176 r_info = ELF64_R_SYM(rela[i].r_info);
177 r_addend = rela[i].r_addend;
178 }
179
180 return _count_relocs;
181}
182
183static int relacmp(const void *_x, const void *_y)
184{
185 const Elf64_Rela *x, *y;
186
187 y = (Elf64_Rela *)_x;
188 x = (Elf64_Rela *)_y;
189
190 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
191 * make the comparison cheaper/faster. It won't affect the sorting or
192 * the counting algorithms' performance
193 */
194 if (x->r_info < y->r_info)
195 return -1;
196 else if (x->r_info > y->r_info)
197 return 1;
198 else if (x->r_addend < y->r_addend)
199 return -1;
200 else if (x->r_addend > y->r_addend)
201 return 1;
202 else
203 return 0;
204}
205
206/* Get size of potential trampolines required. */
207static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
208 const Elf64_Shdr *sechdrs,
209 char *secstrings,
210 struct module *me)
211{
212 /* One extra reloc so it's always 0-addr terminated */
213 unsigned long relocs = 1;
214 unsigned i;
215
216 /* Every relocated section... */
217 for (i = 1; i < hdr->e_shnum; i++) {
218 if (sechdrs[i].sh_type == SHT_RELA) {
219 pr_debug("Found relocations in section %u\n", i);
220 pr_debug("Ptr: %p. Number: %Lu\n",
221 (void *)sechdrs[i].sh_addr,
222 sechdrs[i].sh_size / sizeof(Elf64_Rela));
223
224 /* Sort the relocation information based on a symbol and
225 * addend key. This is a stable O(n*log n) complexity
226 * algorithm but it will reduce the complexity of
227 * count_relocs() to linear complexity O(n)
228 */
229 sort((void *)sechdrs[i].sh_addr,
230 sechdrs[i].sh_size / sizeof(Elf64_Rela),
231 sizeof(Elf64_Rela), relacmp, NULL);
232
233 relocs += count_relocs((void *)sechdrs[i].sh_addr,
234 sechdrs[i].sh_size
235 / sizeof(Elf64_Rela),
236 R_PPC_REL24);
237#ifdef CONFIG_PPC_KERNEL_PCREL
238 relocs += count_relocs((void *)sechdrs[i].sh_addr,
239 sechdrs[i].sh_size
240 / sizeof(Elf64_Rela),
241 R_PPC64_REL24_NOTOC);
242#endif
243 }
244 }
245
246 /* stubs for ftrace_caller and ftrace_regs_caller */
247 relocs += IS_ENABLED(CONFIG_DYNAMIC_FTRACE) + IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS);
248
249#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
250 /* stubs for the function tracer */
251 for (i = 1; i < hdr->e_shnum; i++) {
252 if (!strcmp(secstrings + sechdrs[i].sh_name, "__patchable_function_entries")) {
253 me->arch.ool_stub_count = sechdrs[i].sh_size / sizeof(unsigned long);
254 me->arch.ool_stub_index = 0;
255 relocs += roundup(me->arch.ool_stub_count * sizeof(struct ftrace_ool_stub),
256 sizeof(struct ppc64_stub_entry)) /
257 sizeof(struct ppc64_stub_entry);
258 break;
259 }
260 }
261 if (i == hdr->e_shnum) {
262 pr_err("%s: doesn't contain __patchable_function_entries.\n", me->name);
263 return -ENOEXEC;
264 }
265#endif
266
267 pr_debug("Looks like a total of %lu stubs, max\n", relocs);
268 return relocs * sizeof(struct ppc64_stub_entry);
269}
270
271#ifdef CONFIG_PPC_KERNEL_PCREL
272static int count_pcpu_relocs(const Elf64_Shdr *sechdrs,
273 const Elf64_Rela *rela, unsigned int num,
274 unsigned int symindex, unsigned int pcpu)
275{
276 unsigned int i, r_info, r_addend, _count_relocs;
277
278 _count_relocs = 0;
279 r_info = 0;
280 r_addend = 0;
281
282 for (i = 0; i < num; i++) {
283 Elf64_Sym *sym;
284
285 /* This is the symbol it is referring to */
286 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
287 + ELF64_R_SYM(rela[i].r_info);
288
289 if (sym->st_shndx == pcpu &&
290 (r_info != ELF64_R_SYM(rela[i].r_info) ||
291 r_addend != rela[i].r_addend)) {
292 _count_relocs++;
293 r_info = ELF64_R_SYM(rela[i].r_info);
294 r_addend = rela[i].r_addend;
295 }
296 }
297
298 return _count_relocs;
299}
300
301/* Get size of potential GOT required. */
302static unsigned long get_got_size(const Elf64_Ehdr *hdr,
303 const Elf64_Shdr *sechdrs,
304 struct module *me)
305{
306 /* One extra reloc so it's always 0-addr terminated */
307 unsigned long relocs = 1;
308 unsigned int i, symindex = 0;
309
310 for (i = 1; i < hdr->e_shnum; i++) {
311 if (sechdrs[i].sh_type == SHT_SYMTAB) {
312 symindex = i;
313 break;
314 }
315 }
316 WARN_ON_ONCE(!symindex);
317
318 /* Every relocated section... */
319 for (i = 1; i < hdr->e_shnum; i++) {
320 if (sechdrs[i].sh_type == SHT_RELA) {
321 pr_debug("Found relocations in section %u\n", i);
322 pr_debug("Ptr: %p. Number: %llu\n", (void *)sechdrs[i].sh_addr,
323 sechdrs[i].sh_size / sizeof(Elf64_Rela));
324
325 /*
326 * Sort the relocation information based on a symbol and
327 * addend key. This is a stable O(n*log n) complexity
328 * algorithm but it will reduce the complexity of
329 * count_relocs() to linear complexity O(n)
330 */
331 sort((void *)sechdrs[i].sh_addr,
332 sechdrs[i].sh_size / sizeof(Elf64_Rela),
333 sizeof(Elf64_Rela), relacmp, NULL);
334
335 relocs += count_relocs((void *)sechdrs[i].sh_addr,
336 sechdrs[i].sh_size
337 / sizeof(Elf64_Rela),
338 R_PPC64_GOT_PCREL34);
339
340 /*
341 * Percpu data access typically gets linked with
342 * REL34 relocations, but the percpu section gets
343 * moved at load time and requires that to be
344 * converted to GOT linkage.
345 */
346 if (IS_ENABLED(CONFIG_SMP) && symindex)
347 relocs += count_pcpu_relocs(sechdrs,
348 (void *)sechdrs[i].sh_addr,
349 sechdrs[i].sh_size
350 / sizeof(Elf64_Rela),
351 symindex, me->arch.pcpu_section);
352 }
353 }
354
355 pr_debug("Looks like a total of %lu GOT entries, max\n", relocs);
356 return relocs * sizeof(struct ppc64_got_entry);
357}
358#else /* CONFIG_PPC_KERNEL_PCREL */
359
360/* Still needed for ELFv2, for .TOC. */
361static void dedotify_versions(struct modversion_info *vers,
362 unsigned long size)
363{
364 struct modversion_info *end;
365
366 for (end = (void *)vers + size; vers < end; vers++)
367 if (vers->name[0] == '.') {
368 memmove(vers->name, vers->name+1, strlen(vers->name));
369 }
370}
371
372/*
373 * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC.
374 * seem to be defined (value set later).
375 */
376static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
377{
378 unsigned int i;
379
380 for (i = 1; i < numsyms; i++) {
381 if (syms[i].st_shndx == SHN_UNDEF) {
382 char *name = strtab + syms[i].st_name;
383 if (name[0] == '.') {
384 if (strcmp(name+1, "TOC.") == 0)
385 syms[i].st_shndx = SHN_ABS;
386 syms[i].st_name++;
387 }
388 }
389 }
390}
391
392static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs,
393 const char *strtab,
394 unsigned int symindex)
395{
396 unsigned int i, numsyms;
397 Elf64_Sym *syms;
398
399 syms = (Elf64_Sym *)sechdrs[symindex].sh_addr;
400 numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym);
401
402 for (i = 1; i < numsyms; i++) {
403 if (syms[i].st_shndx == SHN_ABS
404 && strcmp(strtab + syms[i].st_name, "TOC.") == 0)
405 return &syms[i];
406 }
407 return NULL;
408}
409#endif /* CONFIG_PPC_KERNEL_PCREL */
410
411bool module_init_section(const char *name)
412{
413 /* We don't handle .init for the moment: always return false. */
414 return false;
415}
416
417int module_frob_arch_sections(Elf64_Ehdr *hdr,
418 Elf64_Shdr *sechdrs,
419 char *secstrings,
420 struct module *me)
421{
422 unsigned int i;
423
424 /* Find .toc and .stubs sections, symtab and strtab */
425 for (i = 1; i < hdr->e_shnum; i++) {
426 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
427 me->arch.stubs_section = i;
428#ifdef CONFIG_PPC_KERNEL_PCREL
429 else if (strcmp(secstrings + sechdrs[i].sh_name, ".data..percpu") == 0)
430 me->arch.pcpu_section = i;
431 else if (strcmp(secstrings + sechdrs[i].sh_name, ".mygot") == 0) {
432 me->arch.got_section = i;
433 if (sechdrs[i].sh_addralign < 8)
434 sechdrs[i].sh_addralign = 8;
435 }
436#else
437 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) {
438 me->arch.toc_section = i;
439 if (sechdrs[i].sh_addralign < 8)
440 sechdrs[i].sh_addralign = 8;
441 }
442 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
443 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
444 sechdrs[i].sh_size);
445
446 if (sechdrs[i].sh_type == SHT_SYMTAB)
447 dedotify((void *)hdr + sechdrs[i].sh_offset,
448 sechdrs[i].sh_size / sizeof(Elf64_Sym),
449 (void *)hdr
450 + sechdrs[sechdrs[i].sh_link].sh_offset);
451#endif
452 }
453
454 if (!me->arch.stubs_section) {
455 pr_err("%s: doesn't contain .stubs.\n", me->name);
456 return -ENOEXEC;
457 }
458
459#ifdef CONFIG_PPC_KERNEL_PCREL
460 if (!me->arch.got_section) {
461 pr_err("%s: doesn't contain .mygot.\n", me->name);
462 return -ENOEXEC;
463 }
464
465 /* Override the got size */
466 sechdrs[me->arch.got_section].sh_size = get_got_size(hdr, sechdrs, me);
467#else
468 /* If we don't have a .toc, just use .stubs. We need to set r2
469 to some reasonable value in case the module calls out to
470 other functions via a stub, or if a function pointer escapes
471 the module by some means. */
472 if (!me->arch.toc_section)
473 me->arch.toc_section = me->arch.stubs_section;
474#endif
475
476 /* Override the stubs size */
477 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs, secstrings, me);
478
479 return 0;
480}
481
482#if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY)
483
484static u32 stub_insns[] = {
485#ifdef CONFIG_PPC_KERNEL_PCREL
486 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)),
487 PPC_RAW_NOP(), /* align the prefix insn */
488 /* paddi r12,r12,addr */
489 PPC_PREFIX_MLS | __PPC_PRFX_R(0),
490 PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12),
491 PPC_RAW_MTCTR(_R12),
492 PPC_RAW_BCTR(),
493#else
494 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
495 PPC_RAW_ADDIS(_R12, _R12, 0),
496 PPC_RAW_ADDI(_R12, _R12, 0),
497 PPC_RAW_MTCTR(_R12),
498 PPC_RAW_BCTR(),
499#endif
500};
501
502/*
503 * For mprofile-kernel we use a special stub for ftrace_caller() because we
504 * can't rely on r2 containing this module's TOC when we enter the stub.
505 *
506 * That can happen if the function calling us didn't need to use the toc. In
507 * that case it won't have setup r2, and the r2 value will be either the
508 * kernel's toc, or possibly another modules toc.
509 *
510 * To deal with that this stub uses the kernel toc, which is always accessible
511 * via the paca (in r13). The target (ftrace_caller()) is responsible for
512 * saving and restoring the toc before returning.
513 */
514static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
515 unsigned long addr,
516 struct module *me)
517{
518 long reladdr;
519
520 if ((unsigned long)entry->jump % 8 != 0) {
521 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
522 return 0;
523 }
524
525 BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump));
526 memcpy(entry->jump, stub_insns, sizeof(stub_insns));
527
528 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
529 /* Stub uses address relative to kernel base (from the paca) */
530 reladdr = addr - local_paca->kernelbase;
531 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
532 pr_err("%s: Address of %ps out of range of 34-bit relative address.\n",
533 me->name, (void *)addr);
534 return 0;
535 }
536
537 entry->jump[2] |= IMM_H18(reladdr);
538 entry->jump[3] |= IMM_L(reladdr);
539 } else {
540 /* Stub uses address relative to kernel toc (from the paca) */
541 reladdr = addr - kernel_toc_addr();
542 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
543 pr_err("%s: Address of %ps out of range of kernel_toc.\n",
544 me->name, (void *)addr);
545 return 0;
546 }
547
548 entry->jump[1] |= PPC_HA(reladdr);
549 entry->jump[2] |= PPC_LO(reladdr);
550 }
551
552 /* Even though we don't use funcdata in the stub, it's needed elsewhere. */
553 entry->funcdata = func_desc(addr);
554 entry->magic = STUB_MAGIC;
555
556 return 1;
557}
558
559static bool is_mprofile_ftrace_call(const char *name)
560{
561 if (!strcmp("_mcount", name))
562 return true;
563#ifdef CONFIG_DYNAMIC_FTRACE
564 if (!strcmp("ftrace_caller", name))
565 return true;
566#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
567 if (!strcmp("ftrace_regs_caller", name))
568 return true;
569#endif
570#endif
571
572 return false;
573}
574#else
575static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
576 unsigned long addr,
577 struct module *me)
578{
579 return 0;
580}
581
582static bool is_mprofile_ftrace_call(const char *name)
583{
584 return false;
585}
586#endif
587
588/*
589 * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
590 * value maximum span in an instruction which uses a signed offset). Round down
591 * to a 256 byte boundary for the odd case where we are setting up r2 without a
592 * .toc section.
593 */
594static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me)
595{
596#ifndef CONFIG_PPC_KERNEL_PCREL
597 return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
598#else
599 return -1;
600#endif
601}
602
603/* Patch stub to reference function and correct r2 value. */
604static inline int create_stub(const Elf64_Shdr *sechdrs,
605 struct ppc64_stub_entry *entry,
606 unsigned long addr,
607 struct module *me,
608 const char *name)
609{
610 long reladdr;
611 func_desc_t desc;
612 int i;
613
614 if (is_mprofile_ftrace_call(name))
615 return create_ftrace_stub(entry, addr, me);
616
617 if ((unsigned long)entry->jump % 8 != 0) {
618 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
619 return 0;
620 }
621
622 BUILD_BUG_ON(sizeof(ppc64_stub_insns) > sizeof(entry->jump));
623 for (i = 0; i < ARRAY_SIZE(ppc64_stub_insns); i++) {
624 if (patch_instruction(&entry->jump[i],
625 ppc_inst(ppc64_stub_insns[i])))
626 return 0;
627 }
628
629 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
630 /* Stub uses address relative to itself! */
631 reladdr = 0 + offsetof(struct ppc64_stub_entry, funcdata);
632 BUILD_BUG_ON(reladdr != 32);
633 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
634 pr_err("%s: Address of %p out of range of 34-bit relative address.\n",
635 me->name, (void *)reladdr);
636 return 0;
637 }
638 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
639
640 /* May not even need this if we're relative to 0 */
641 if (patch_instruction(&entry->jump[0],
642 ppc_inst_prefix(entry->jump[0] | IMM_H18(reladdr),
643 entry->jump[1] | IMM_L(reladdr))))
644 return 0;
645
646 } else {
647 /* Stub uses address relative to r2. */
648 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
649 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
650 pr_err("%s: Address %p of stub out of range of %p.\n",
651 me->name, (void *)reladdr, (void *)my_r2);
652 return 0;
653 }
654 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr);
655
656 if (patch_instruction(&entry->jump[0],
657 ppc_inst(entry->jump[0] | PPC_HA(reladdr))))
658 return 0;
659
660 if (patch_instruction(&entry->jump[1],
661 ppc_inst(entry->jump[1] | PPC_LO(reladdr))))
662 return 0;
663 }
664
665 // func_desc_t is 8 bytes if ABIv2, else 16 bytes
666 desc = func_desc(addr);
667 for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) {
668 if (patch_u32(((u32 *)&entry->funcdata) + i, ((u32 *)&desc)[i]))
669 return 0;
670 }
671
672 if (patch_u32(&entry->magic, STUB_MAGIC))
673 return 0;
674
675 return 1;
676}
677
678/* Create stub to jump to function described in this OPD/ptr: we need the
679 stub to set up the TOC ptr (r2) for the function. */
680static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
681 unsigned long addr,
682 struct module *me,
683 const char *name)
684{
685 struct ppc64_stub_entry *stubs;
686 unsigned int i, num_stubs;
687
688 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
689
690 /* Find this stub, or if that fails, the next avail. entry */
691 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
692 for (i = 0; stub_func_addr(stubs[i].funcdata); i++) {
693 if (WARN_ON(i >= num_stubs))
694 return 0;
695
696 if (stub_func_addr(stubs[i].funcdata) == func_addr(addr))
697 return (unsigned long)&stubs[i];
698 }
699
700 if (!create_stub(sechdrs, &stubs[i], addr, me, name))
701 return 0;
702
703 return (unsigned long)&stubs[i];
704}
705
706#ifdef CONFIG_PPC_KERNEL_PCREL
707/* Create GOT to load the location described in this ptr */
708static unsigned long got_for_addr(const Elf64_Shdr *sechdrs,
709 unsigned long addr,
710 struct module *me,
711 const char *name)
712{
713 struct ppc64_got_entry *got;
714 unsigned int i, num_got;
715
716 if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
717 return addr;
718
719 num_got = sechdrs[me->arch.got_section].sh_size / sizeof(*got);
720
721 /* Find this stub, or if that fails, the next avail. entry */
722 got = (void *)sechdrs[me->arch.got_section].sh_addr;
723 for (i = 0; got[i].addr; i++) {
724 if (WARN_ON(i >= num_got))
725 return 0;
726
727 if (got[i].addr == addr)
728 return (unsigned long)&got[i];
729 }
730
731 got[i].addr = addr;
732
733 return (unsigned long)&got[i];
734}
735#endif
736
737/* We expect a noop next: if it is, replace it with instruction to
738 restore r2. */
739static int restore_r2(const char *name, u32 *instruction, struct module *me)
740{
741 u32 *prev_insn = instruction - 1;
742 u32 insn_val = *instruction;
743
744 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
745 return 0;
746
747 if (is_mprofile_ftrace_call(name))
748 return 0;
749
750 /*
751 * Make sure the branch isn't a sibling call. Sibling calls aren't
752 * "link" branches and they don't return, so they don't need the r2
753 * restore afterwards.
754 */
755 if (!instr_is_relative_link_branch(ppc_inst(*prev_insn)))
756 return 0;
757
758 /*
759 * For livepatch, the restore r2 instruction might have already been
760 * written previously, if the referenced symbol is in a previously
761 * unloaded module which is now being loaded again. In that case, skip
762 * the warning and the instruction write.
763 */
764 if (insn_val == PPC_INST_LD_TOC)
765 return 0;
766
767 if (insn_val != PPC_RAW_NOP()) {
768 pr_err("%s: Expected nop after call, got %08x at %pS\n",
769 me->name, insn_val, instruction);
770 return -ENOEXEC;
771 }
772
773 /* ld r2,R2_STACK_OFFSET(r1) */
774 return patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC));
775}
776
777int apply_relocate_add(Elf64_Shdr *sechdrs,
778 const char *strtab,
779 unsigned int symindex,
780 unsigned int relsec,
781 struct module *me)
782{
783 unsigned int i;
784 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
785 Elf64_Sym *sym;
786 unsigned long *location;
787 unsigned long value;
788
789 pr_debug("Applying ADD relocate section %u to %u\n", relsec,
790 sechdrs[relsec].sh_info);
791
792#ifndef CONFIG_PPC_KERNEL_PCREL
793 /* First time we're called, we can fix up .TOC. */
794 if (!me->arch.toc_fixed) {
795 sym = find_dot_toc(sechdrs, strtab, symindex);
796 /* It's theoretically possible that a module doesn't want a
797 * .TOC. so don't fail it just for that. */
798 if (sym)
799 sym->st_value = my_r2(sechdrs, me);
800 me->arch.toc_fixed = true;
801 }
802#endif
803 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
804 /* This is where to make the change */
805 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
806 + rela[i].r_offset;
807 /* This is the symbol it is referring to */
808 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
809 + ELF64_R_SYM(rela[i].r_info);
810
811 pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n",
812 location, (long)ELF64_R_TYPE(rela[i].r_info),
813 strtab + sym->st_name, (unsigned long)sym->st_value,
814 (long)rela[i].r_addend);
815
816 /* `Everything is relative'. */
817 value = sym->st_value + rela[i].r_addend;
818
819 switch (ELF64_R_TYPE(rela[i].r_info)) {
820 case R_PPC64_ADDR32:
821 /* Simply set it */
822 *(u32 *)location = value;
823 break;
824
825 case R_PPC64_ADDR64:
826 /* Simply set it */
827 *(unsigned long *)location = value;
828 break;
829
830#ifndef CONFIG_PPC_KERNEL_PCREL
831 case R_PPC64_TOC:
832 *(unsigned long *)location = my_r2(sechdrs, me);
833 break;
834
835 case R_PPC64_TOC16:
836 /* Subtract TOC pointer */
837 value -= my_r2(sechdrs, me);
838 if (value + 0x8000 > 0xffff) {
839 pr_err("%s: bad TOC16 relocation (0x%lx)\n",
840 me->name, value);
841 return -ENOEXEC;
842 }
843 *((uint16_t *) location)
844 = (*((uint16_t *) location) & ~0xffff)
845 | (value & 0xffff);
846 break;
847
848 case R_PPC64_TOC16_LO:
849 /* Subtract TOC pointer */
850 value -= my_r2(sechdrs, me);
851 *((uint16_t *) location)
852 = (*((uint16_t *) location) & ~0xffff)
853 | (value & 0xffff);
854 break;
855
856 case R_PPC64_TOC16_DS:
857 /* Subtract TOC pointer */
858 value -= my_r2(sechdrs, me);
859 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
860 pr_err("%s: bad TOC16_DS relocation (0x%lx)\n",
861 me->name, value);
862 return -ENOEXEC;
863 }
864 *((uint16_t *) location)
865 = (*((uint16_t *) location) & ~0xfffc)
866 | (value & 0xfffc);
867 break;
868
869 case R_PPC64_TOC16_LO_DS:
870 /* Subtract TOC pointer */
871 value -= my_r2(sechdrs, me);
872 if ((value & 3) != 0) {
873 pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n",
874 me->name, value);
875 return -ENOEXEC;
876 }
877 *((uint16_t *) location)
878 = (*((uint16_t *) location) & ~0xfffc)
879 | (value & 0xfffc);
880 break;
881
882 case R_PPC64_TOC16_HA:
883 /* Subtract TOC pointer */
884 value -= my_r2(sechdrs, me);
885 value = ((value + 0x8000) >> 16);
886 *((uint16_t *) location)
887 = (*((uint16_t *) location) & ~0xffff)
888 | (value & 0xffff);
889 break;
890#endif
891
892 case R_PPC_REL24:
893#ifdef CONFIG_PPC_KERNEL_PCREL
894 /* PCREL still generates REL24 for mcount */
895 case R_PPC64_REL24_NOTOC:
896#endif
897 /* FIXME: Handle weak symbols here --RR */
898 if (sym->st_shndx == SHN_UNDEF ||
899 sym->st_shndx == SHN_LIVEPATCH) {
900 /* External: go via stub */
901 value = stub_for_addr(sechdrs, value, me,
902 strtab + sym->st_name);
903 if (!value)
904 return -ENOENT;
905 if (restore_r2(strtab + sym->st_name,
906 (u32 *)location + 1, me))
907 return -ENOEXEC;
908 } else
909 value += local_entry_offset(sym);
910
911 /* Convert value to relative */
912 value -= (unsigned long)location;
913 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
914 pr_err("%s: REL24 %li out of range!\n",
915 me->name, (long int)value);
916 return -ENOEXEC;
917 }
918
919 /* Only replace bits 2 through 26 */
920 value = (*(uint32_t *)location & ~PPC_LI_MASK) | PPC_LI(value);
921
922 if (patch_instruction((u32 *)location, ppc_inst(value)))
923 return -EFAULT;
924
925 break;
926
927 case R_PPC64_REL64:
928 /* 64 bits relative (used by features fixups) */
929 *location = value - (unsigned long)location;
930 break;
931
932 case R_PPC64_REL32:
933 /* 32 bits relative (used by relative exception tables) */
934 /* Convert value to relative */
935 value -= (unsigned long)location;
936 if (value + 0x80000000 > 0xffffffff) {
937 pr_err("%s: REL32 %li out of range!\n",
938 me->name, (long int)value);
939 return -ENOEXEC;
940 }
941 *(u32 *)location = value;
942 break;
943
944#ifdef CONFIG_PPC_KERNEL_PCREL
945 case R_PPC64_PCREL34: {
946 unsigned long absvalue = value;
947
948 /* Convert value to relative */
949 value -= (unsigned long)location;
950
951 if (value + 0x200000000 > 0x3ffffffff) {
952 if (sym->st_shndx != me->arch.pcpu_section) {
953 pr_err("%s: REL34 %li out of range!\n",
954 me->name, (long)value);
955 return -ENOEXEC;
956 }
957
958 /*
959 * per-cpu section is special cased because
960 * it is moved during loading, so has to be
961 * converted to use GOT.
962 */
963 value = got_for_addr(sechdrs, absvalue, me,
964 strtab + sym->st_name);
965 if (!value)
966 return -ENOENT;
967 value -= (unsigned long)location;
968
969 /* Turn pla into pld */
970 if (patch_instruction((u32 *)location,
971 ppc_inst_prefix((*(u32 *)location & ~0x02000000),
972 (*((u32 *)location + 1) & ~0xf8000000) | 0xe4000000)))
973 return -EFAULT;
974 }
975
976 if (patch_instruction((u32 *)location,
977 ppc_inst_prefix((*(u32 *)location & ~0x3ffff) | IMM_H18(value),
978 (*((u32 *)location + 1) & ~0xffff) | IMM_L(value))))
979 return -EFAULT;
980
981 break;
982 }
983
984#else
985 case R_PPC64_TOCSAVE:
986 /*
987 * Marker reloc indicates we don't have to save r2.
988 * That would only save us one instruction, so ignore
989 * it.
990 */
991 break;
992#endif
993
994 case R_PPC64_ENTRY:
995 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
996 break;
997
998 /*
999 * Optimize ELFv2 large code model entry point if
1000 * the TOC is within 2GB range of current location.
1001 */
1002 value = my_r2(sechdrs, me) - (unsigned long)location;
1003 if (value + 0x80008000 > 0xffffffff)
1004 break;
1005 /*
1006 * Check for the large code model prolog sequence:
1007 * ld r2, ...(r12)
1008 * add r2, r2, r12
1009 */
1010 if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0))
1011 break;
1012 if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12))
1013 break;
1014 /*
1015 * If found, replace it with:
1016 * addis r2, r12, (.TOC.-func)@ha
1017 * addi r2, r2, (.TOC.-func)@l
1018 */
1019 ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value));
1020 ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value));
1021 break;
1022
1023 case R_PPC64_REL16_HA:
1024 /* Subtract location pointer */
1025 value -= (unsigned long)location;
1026 value = ((value + 0x8000) >> 16);
1027 *((uint16_t *) location)
1028 = (*((uint16_t *) location) & ~0xffff)
1029 | (value & 0xffff);
1030 break;
1031
1032 case R_PPC64_REL16_LO:
1033 /* Subtract location pointer */
1034 value -= (unsigned long)location;
1035 *((uint16_t *) location)
1036 = (*((uint16_t *) location) & ~0xffff)
1037 | (value & 0xffff);
1038 break;
1039
1040#ifdef CONFIG_PPC_KERNEL_PCREL
1041 case R_PPC64_GOT_PCREL34:
1042 value = got_for_addr(sechdrs, value, me,
1043 strtab + sym->st_name);
1044 if (!value)
1045 return -ENOENT;
1046 value -= (unsigned long)location;
1047 ((uint32_t *)location)[0] = (((uint32_t *)location)[0] & ~0x3ffff) |
1048 ((value >> 16) & 0x3ffff);
1049 ((uint32_t *)location)[1] = (((uint32_t *)location)[1] & ~0xffff) |
1050 (value & 0xffff);
1051 break;
1052#endif
1053
1054 default:
1055 pr_err("%s: Unknown ADD relocation: %lu\n",
1056 me->name,
1057 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
1058 return -ENOEXEC;
1059 }
1060 }
1061
1062 return 0;
1063}
1064
1065#ifdef CONFIG_DYNAMIC_FTRACE
1066int module_trampoline_target(struct module *mod, unsigned long addr,
1067 unsigned long *target)
1068{
1069 struct ppc64_stub_entry *stub;
1070 func_desc_t funcdata;
1071 u32 magic;
1072
1073 if (!within_module_core(addr, mod)) {
1074 pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
1075 return -EFAULT;
1076 }
1077
1078 stub = (struct ppc64_stub_entry *)addr;
1079
1080 if (copy_from_kernel_nofault(&magic, &stub->magic,
1081 sizeof(magic))) {
1082 pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
1083 return -EFAULT;
1084 }
1085
1086 if (magic != STUB_MAGIC) {
1087 pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
1088 return -EFAULT;
1089 }
1090
1091 if (copy_from_kernel_nofault(&funcdata, &stub->funcdata,
1092 sizeof(funcdata))) {
1093 pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
1094 return -EFAULT;
1095 }
1096
1097 *target = stub_func_addr(funcdata);
1098
1099 return 0;
1100}
1101
1102static int setup_ftrace_ool_stubs(const Elf64_Shdr *sechdrs, unsigned long addr, struct module *me)
1103{
1104#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
1105 unsigned int i, total_stubs, num_stubs;
1106 struct ppc64_stub_entry *stub;
1107
1108 total_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stub);
1109 num_stubs = roundup(me->arch.ool_stub_count * sizeof(struct ftrace_ool_stub),
1110 sizeof(struct ppc64_stub_entry)) / sizeof(struct ppc64_stub_entry);
1111
1112 /* Find the next available entry */
1113 stub = (void *)sechdrs[me->arch.stubs_section].sh_addr;
1114 for (i = 0; stub_func_addr(stub[i].funcdata); i++)
1115 if (WARN_ON(i >= total_stubs))
1116 return -1;
1117
1118 if (WARN_ON(i + num_stubs > total_stubs))
1119 return -1;
1120
1121 stub += i;
1122 me->arch.ool_stubs = (struct ftrace_ool_stub *)stub;
1123
1124 /* reserve stubs */
1125 for (i = 0; i < num_stubs; i++)
1126 if (patch_u32((void *)&stub->funcdata, PPC_RAW_NOP()))
1127 return -1;
1128#endif
1129
1130 return 0;
1131}
1132
1133int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
1134{
1135 mod->arch.tramp = stub_for_addr(sechdrs,
1136 (unsigned long)ftrace_caller,
1137 mod,
1138 "ftrace_caller");
1139#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1140 mod->arch.tramp_regs = stub_for_addr(sechdrs,
1141 (unsigned long)ftrace_regs_caller,
1142 mod,
1143 "ftrace_regs_caller");
1144 if (!mod->arch.tramp_regs)
1145 return -ENOENT;
1146#endif
1147
1148 if (!mod->arch.tramp)
1149 return -ENOENT;
1150
1151 if (setup_ftrace_ool_stubs(sechdrs, mod->arch.tramp, mod))
1152 return -ENOENT;
1153
1154 return 0;
1155}
1156#endif
1/* Kernel module help for PPC64.
2 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17*/
18#include <linux/module.h>
19#include <linux/elf.h>
20#include <linux/moduleloader.h>
21#include <linux/err.h>
22#include <linux/vmalloc.h>
23#include <linux/ftrace.h>
24#include <linux/bug.h>
25#include <asm/module.h>
26#include <asm/firmware.h>
27#include <asm/code-patching.h>
28#include <linux/sort.h>
29#include <asm/setup.h>
30
31/* FIXME: We don't do .init separately. To do this, we'd need to have
32 a separate r2 value in the init and core section, and stub between
33 them, too.
34
35 Using a magic allocator which places modules within 32MB solves
36 this, and makes other things simpler. Anton?
37 --RR. */
38#if 0
39#define DEBUGP printk
40#else
41#define DEBUGP(fmt , ...)
42#endif
43
44/* Like PPC32, we need little trampolines to do > 24-bit jumps (into
45 the kernel itself). But on PPC64, these need to be used for every
46 jump, actually, to reset r2 (TOC+0x8000). */
47struct ppc64_stub_entry
48{
49 /* 28 byte jump instruction sequence (7 instructions) */
50 unsigned char jump[28];
51 unsigned char unused[4];
52 /* Data for the above code */
53 struct ppc64_opd_entry opd;
54};
55
56/* We use a stub to fix up r2 (TOC ptr) and to jump to the (external)
57 function which may be more than 24-bits away. We could simply
58 patch the new r2 value and function pointer into the stub, but it's
59 significantly shorter to put these values at the end of the stub
60 code, and patch the stub address (32-bits relative to the TOC ptr,
61 r2) into the stub. */
62static struct ppc64_stub_entry ppc64_stub =
63{ .jump = {
64#ifdef __LITTLE_ENDIAN__
65 0x00, 0x00, 0x82, 0x3d, /* addis r12,r2, <high> */
66 0x00, 0x00, 0x8c, 0x39, /* addi r12,r12, <low> */
67 /* Save current r2 value in magic place on the stack. */
68 0x28, 0x00, 0x41, 0xf8, /* std r2,40(r1) */
69 0x20, 0x00, 0x6c, 0xe9, /* ld r11,32(r12) */
70 0x28, 0x00, 0x4c, 0xe8, /* ld r2,40(r12) */
71 0xa6, 0x03, 0x69, 0x7d, /* mtctr r11 */
72 0x20, 0x04, 0x80, 0x4e /* bctr */
73#else
74 0x3d, 0x82, 0x00, 0x00, /* addis r12,r2, <high> */
75 0x39, 0x8c, 0x00, 0x00, /* addi r12,r12, <low> */
76 /* Save current r2 value in magic place on the stack. */
77 0xf8, 0x41, 0x00, 0x28, /* std r2,40(r1) */
78 0xe9, 0x6c, 0x00, 0x20, /* ld r11,32(r12) */
79 0xe8, 0x4c, 0x00, 0x28, /* ld r2,40(r12) */
80 0x7d, 0x69, 0x03, 0xa6, /* mtctr r11 */
81 0x4e, 0x80, 0x04, 0x20 /* bctr */
82#endif
83} };
84
85/* Count how many different 24-bit relocations (different symbol,
86 different addend) */
87static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num)
88{
89 unsigned int i, r_info, r_addend, _count_relocs;
90
91 /* FIXME: Only count external ones --RR */
92 _count_relocs = 0;
93 r_info = 0;
94 r_addend = 0;
95 for (i = 0; i < num; i++)
96 /* Only count 24-bit relocs, others don't need stubs */
97 if (ELF64_R_TYPE(rela[i].r_info) == R_PPC_REL24 &&
98 (r_info != ELF64_R_SYM(rela[i].r_info) ||
99 r_addend != rela[i].r_addend)) {
100 _count_relocs++;
101 r_info = ELF64_R_SYM(rela[i].r_info);
102 r_addend = rela[i].r_addend;
103 }
104
105 return _count_relocs;
106}
107
108static int relacmp(const void *_x, const void *_y)
109{
110 const Elf64_Rela *x, *y;
111
112 y = (Elf64_Rela *)_x;
113 x = (Elf64_Rela *)_y;
114
115 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
116 * make the comparison cheaper/faster. It won't affect the sorting or
117 * the counting algorithms' performance
118 */
119 if (x->r_info < y->r_info)
120 return -1;
121 else if (x->r_info > y->r_info)
122 return 1;
123 else if (x->r_addend < y->r_addend)
124 return -1;
125 else if (x->r_addend > y->r_addend)
126 return 1;
127 else
128 return 0;
129}
130
131static void relaswap(void *_x, void *_y, int size)
132{
133 uint64_t *x, *y, tmp;
134 int i;
135
136 y = (uint64_t *)_x;
137 x = (uint64_t *)_y;
138
139 for (i = 0; i < sizeof(Elf64_Rela) / sizeof(uint64_t); i++) {
140 tmp = x[i];
141 x[i] = y[i];
142 y[i] = tmp;
143 }
144}
145
146/* Get size of potential trampolines required. */
147static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
148 const Elf64_Shdr *sechdrs)
149{
150 /* One extra reloc so it's always 0-funcaddr terminated */
151 unsigned long relocs = 1;
152 unsigned i;
153
154 /* Every relocated section... */
155 for (i = 1; i < hdr->e_shnum; i++) {
156 if (sechdrs[i].sh_type == SHT_RELA) {
157 DEBUGP("Found relocations in section %u\n", i);
158 DEBUGP("Ptr: %p. Number: %lu\n",
159 (void *)sechdrs[i].sh_addr,
160 sechdrs[i].sh_size / sizeof(Elf64_Rela));
161
162 /* Sort the relocation information based on a symbol and
163 * addend key. This is a stable O(n*log n) complexity
164 * alogrithm but it will reduce the complexity of
165 * count_relocs() to linear complexity O(n)
166 */
167 sort((void *)sechdrs[i].sh_addr,
168 sechdrs[i].sh_size / sizeof(Elf64_Rela),
169 sizeof(Elf64_Rela), relacmp, relaswap);
170
171 relocs += count_relocs((void *)sechdrs[i].sh_addr,
172 sechdrs[i].sh_size
173 / sizeof(Elf64_Rela));
174 }
175 }
176
177#ifdef CONFIG_DYNAMIC_FTRACE
178 /* make the trampoline to the ftrace_caller */
179 relocs++;
180#endif
181
182 DEBUGP("Looks like a total of %lu stubs, max\n", relocs);
183 return relocs * sizeof(struct ppc64_stub_entry);
184}
185
186static void dedotify_versions(struct modversion_info *vers,
187 unsigned long size)
188{
189 struct modversion_info *end;
190
191 for (end = (void *)vers + size; vers < end; vers++)
192 if (vers->name[0] == '.')
193 memmove(vers->name, vers->name+1, strlen(vers->name));
194}
195
196/* Undefined symbols which refer to .funcname, hack to funcname */
197static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
198{
199 unsigned int i;
200
201 for (i = 1; i < numsyms; i++) {
202 if (syms[i].st_shndx == SHN_UNDEF) {
203 char *name = strtab + syms[i].st_name;
204 if (name[0] == '.')
205 memmove(name, name+1, strlen(name));
206 }
207 }
208}
209
210int module_frob_arch_sections(Elf64_Ehdr *hdr,
211 Elf64_Shdr *sechdrs,
212 char *secstrings,
213 struct module *me)
214{
215 unsigned int i;
216
217 /* Find .toc and .stubs sections, symtab and strtab */
218 for (i = 1; i < hdr->e_shnum; i++) {
219 char *p;
220 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
221 me->arch.stubs_section = i;
222 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0)
223 me->arch.toc_section = i;
224 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
225 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
226 sechdrs[i].sh_size);
227
228 /* We don't handle .init for the moment: rename to _init */
229 while ((p = strstr(secstrings + sechdrs[i].sh_name, ".init")))
230 p[0] = '_';
231
232 if (sechdrs[i].sh_type == SHT_SYMTAB)
233 dedotify((void *)hdr + sechdrs[i].sh_offset,
234 sechdrs[i].sh_size / sizeof(Elf64_Sym),
235 (void *)hdr
236 + sechdrs[sechdrs[i].sh_link].sh_offset);
237 }
238
239 if (!me->arch.stubs_section) {
240 printk("%s: doesn't contain .stubs.\n", me->name);
241 return -ENOEXEC;
242 }
243
244 /* If we don't have a .toc, just use .stubs. We need to set r2
245 to some reasonable value in case the module calls out to
246 other functions via a stub, or if a function pointer escapes
247 the module by some means. */
248 if (!me->arch.toc_section)
249 me->arch.toc_section = me->arch.stubs_section;
250
251 /* Override the stubs size */
252 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
253 return 0;
254}
255
256/* r2 is the TOC pointer: it actually points 0x8000 into the TOC (this
257 gives the value maximum span in an instruction which uses a signed
258 offset) */
259static inline unsigned long my_r2(Elf64_Shdr *sechdrs, struct module *me)
260{
261 return sechdrs[me->arch.toc_section].sh_addr + 0x8000;
262}
263
264/* Both low and high 16 bits are added as SIGNED additions, so if low
265 16 bits has high bit set, high 16 bits must be adjusted. These
266 macros do that (stolen from binutils). */
267#define PPC_LO(v) ((v) & 0xffff)
268#define PPC_HI(v) (((v) >> 16) & 0xffff)
269#define PPC_HA(v) PPC_HI ((v) + 0x8000)
270
271/* Patch stub to reference function and correct r2 value. */
272static inline int create_stub(Elf64_Shdr *sechdrs,
273 struct ppc64_stub_entry *entry,
274 struct ppc64_opd_entry *opd,
275 struct module *me)
276{
277 Elf64_Half *loc1, *loc2;
278 long reladdr;
279
280 *entry = ppc64_stub;
281
282#ifdef __LITTLE_ENDIAN__
283 loc1 = (Elf64_Half *)&entry->jump[0];
284 loc2 = (Elf64_Half *)&entry->jump[4];
285#else
286 loc1 = (Elf64_Half *)&entry->jump[2];
287 loc2 = (Elf64_Half *)&entry->jump[6];
288#endif
289
290 /* Stub uses address relative to r2. */
291 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
292 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
293 printk("%s: Address %p of stub out of range of %p.\n",
294 me->name, (void *)reladdr, (void *)my_r2);
295 return 0;
296 }
297 DEBUGP("Stub %p get data from reladdr %li\n", entry, reladdr);
298
299 *loc1 = PPC_HA(reladdr);
300 *loc2 = PPC_LO(reladdr);
301 entry->opd.funcaddr = opd->funcaddr;
302 entry->opd.r2 = opd->r2;
303 return 1;
304}
305
306/* Create stub to jump to function described in this OPD: we need the
307 stub to set up the TOC ptr (r2) for the function. */
308static unsigned long stub_for_addr(Elf64_Shdr *sechdrs,
309 unsigned long opdaddr,
310 struct module *me)
311{
312 struct ppc64_stub_entry *stubs;
313 struct ppc64_opd_entry *opd = (void *)opdaddr;
314 unsigned int i, num_stubs;
315
316 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
317
318 /* Find this stub, or if that fails, the next avail. entry */
319 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
320 for (i = 0; stubs[i].opd.funcaddr; i++) {
321 BUG_ON(i >= num_stubs);
322
323 if (stubs[i].opd.funcaddr == opd->funcaddr)
324 return (unsigned long)&stubs[i];
325 }
326
327 if (!create_stub(sechdrs, &stubs[i], opd, me))
328 return 0;
329
330 return (unsigned long)&stubs[i];
331}
332
333/* We expect a noop next: if it is, replace it with instruction to
334 restore r2. */
335static int restore_r2(u32 *instruction, struct module *me)
336{
337 if (*instruction != PPC_INST_NOP) {
338 printk("%s: Expect noop after relocate, got %08x\n",
339 me->name, *instruction);
340 return 0;
341 }
342 *instruction = 0xe8410028; /* ld r2,40(r1) */
343 return 1;
344}
345
346int apply_relocate_add(Elf64_Shdr *sechdrs,
347 const char *strtab,
348 unsigned int symindex,
349 unsigned int relsec,
350 struct module *me)
351{
352 unsigned int i;
353 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
354 Elf64_Sym *sym;
355 unsigned long *location;
356 unsigned long value;
357
358 DEBUGP("Applying ADD relocate section %u to %u\n", relsec,
359 sechdrs[relsec].sh_info);
360 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
361 /* This is where to make the change */
362 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
363 + rela[i].r_offset;
364 /* This is the symbol it is referring to */
365 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
366 + ELF64_R_SYM(rela[i].r_info);
367
368 DEBUGP("RELOC at %p: %li-type as %s (%lu) + %li\n",
369 location, (long)ELF64_R_TYPE(rela[i].r_info),
370 strtab + sym->st_name, (unsigned long)sym->st_value,
371 (long)rela[i].r_addend);
372
373 /* `Everything is relative'. */
374 value = sym->st_value + rela[i].r_addend;
375
376 switch (ELF64_R_TYPE(rela[i].r_info)) {
377 case R_PPC64_ADDR32:
378 /* Simply set it */
379 *(u32 *)location = value;
380 break;
381
382 case R_PPC64_ADDR64:
383 /* Simply set it */
384 *(unsigned long *)location = value;
385 break;
386
387 case R_PPC64_TOC:
388 *(unsigned long *)location = my_r2(sechdrs, me);
389 break;
390
391 case R_PPC64_TOC16:
392 /* Subtract TOC pointer */
393 value -= my_r2(sechdrs, me);
394 if (value + 0x8000 > 0xffff) {
395 printk("%s: bad TOC16 relocation (%lu)\n",
396 me->name, value);
397 return -ENOEXEC;
398 }
399 *((uint16_t *) location)
400 = (*((uint16_t *) location) & ~0xffff)
401 | (value & 0xffff);
402 break;
403
404 case R_PPC64_TOC16_LO:
405 /* Subtract TOC pointer */
406 value -= my_r2(sechdrs, me);
407 *((uint16_t *) location)
408 = (*((uint16_t *) location) & ~0xffff)
409 | (value & 0xffff);
410 break;
411
412 case R_PPC64_TOC16_DS:
413 /* Subtract TOC pointer */
414 value -= my_r2(sechdrs, me);
415 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
416 printk("%s: bad TOC16_DS relocation (%lu)\n",
417 me->name, value);
418 return -ENOEXEC;
419 }
420 *((uint16_t *) location)
421 = (*((uint16_t *) location) & ~0xfffc)
422 | (value & 0xfffc);
423 break;
424
425 case R_PPC64_TOC16_LO_DS:
426 /* Subtract TOC pointer */
427 value -= my_r2(sechdrs, me);
428 if ((value & 3) != 0) {
429 printk("%s: bad TOC16_LO_DS relocation (%lu)\n",
430 me->name, value);
431 return -ENOEXEC;
432 }
433 *((uint16_t *) location)
434 = (*((uint16_t *) location) & ~0xfffc)
435 | (value & 0xfffc);
436 break;
437
438 case R_PPC64_TOC16_HA:
439 /* Subtract TOC pointer */
440 value -= my_r2(sechdrs, me);
441 value = ((value + 0x8000) >> 16);
442 *((uint16_t *) location)
443 = (*((uint16_t *) location) & ~0xffff)
444 | (value & 0xffff);
445 break;
446
447 case R_PPC_REL24:
448 /* FIXME: Handle weak symbols here --RR */
449 if (sym->st_shndx == SHN_UNDEF) {
450 /* External: go via stub */
451 value = stub_for_addr(sechdrs, value, me);
452 if (!value)
453 return -ENOENT;
454 if (!restore_r2((u32 *)location + 1, me))
455 return -ENOEXEC;
456 }
457
458 /* Convert value to relative */
459 value -= (unsigned long)location;
460 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
461 printk("%s: REL24 %li out of range!\n",
462 me->name, (long int)value);
463 return -ENOEXEC;
464 }
465
466 /* Only replace bits 2 through 26 */
467 *(uint32_t *)location
468 = (*(uint32_t *)location & ~0x03fffffc)
469 | (value & 0x03fffffc);
470 break;
471
472 case R_PPC64_REL64:
473 /* 64 bits relative (used by features fixups) */
474 *location = value - (unsigned long)location;
475 break;
476
477 default:
478 printk("%s: Unknown ADD relocation: %lu\n",
479 me->name,
480 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
481 return -ENOEXEC;
482 }
483 }
484
485#ifdef CONFIG_DYNAMIC_FTRACE
486 me->arch.toc = my_r2(sechdrs, me);
487 me->arch.tramp = stub_for_addr(sechdrs,
488 (unsigned long)ftrace_caller,
489 me);
490#endif
491
492 return 0;
493}