Loading...
Note: File does not exist in v6.8.
1/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
7 *
8 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
10 *
11 */
12
13#include <linux/spinlock.h>
14#include <linux/hardirq.h>
15#include <linux/uaccess.h>
16#include <linux/module.h>
17#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/init.h>
20#include <linux/list.h>
21
22#include <asm/cacheflush.h>
23#include <asm/code-patching.h>
24#include <asm/ftrace.h>
25#include <asm/syscall.h>
26
27
28#ifdef CONFIG_DYNAMIC_FTRACE
29static unsigned int
30ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
31{
32 unsigned int op;
33
34 addr = ppc_function_entry((void *)addr);
35
36 /* if (link) set op to 'bl' else 'b' */
37 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
38
39 return op;
40}
41
42static int
43ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
44{
45 unsigned int replaced;
46
47 /*
48 * Note: Due to modules and __init, code can
49 * disappear and change, we need to protect against faulting
50 * as well as code changing. We do this by using the
51 * probe_kernel_* functions.
52 *
53 * No real locking needed, this code is run through
54 * kstop_machine, or before SMP starts.
55 */
56
57 /* read the text we want to modify */
58 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
59 return -EFAULT;
60
61 /* Make sure it is what we expect it to be */
62 if (replaced != old)
63 return -EINVAL;
64
65 /* replace the text with the new text */
66 if (patch_instruction((unsigned int *)ip, new))
67 return -EPERM;
68
69 return 0;
70}
71
72/*
73 * Helper functions that are the same for both PPC64 and PPC32.
74 */
75static int test_24bit_addr(unsigned long ip, unsigned long addr)
76{
77 addr = ppc_function_entry((void *)addr);
78
79 /* use the create_branch to verify that this offset can be branched */
80 return create_branch((unsigned int *)ip, addr, 0);
81}
82
83#ifdef CONFIG_MODULES
84
85static int is_bl_op(unsigned int op)
86{
87 return (op & 0xfc000003) == 0x48000001;
88}
89
90static unsigned long find_bl_target(unsigned long ip, unsigned int op)
91{
92 static int offset;
93
94 offset = (op & 0x03fffffc);
95 /* make it signed */
96 if (offset & 0x02000000)
97 offset |= 0xfe000000;
98
99 return ip + (long)offset;
100}
101
102#ifdef CONFIG_PPC64
103static int
104__ftrace_make_nop(struct module *mod,
105 struct dyn_ftrace *rec, unsigned long addr)
106{
107 unsigned int op;
108 unsigned int jmp[5];
109 unsigned long ptr;
110 unsigned long ip = rec->ip;
111 unsigned long tramp;
112 int offset;
113
114 /* read where this goes */
115 if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
116 return -EFAULT;
117
118 /* Make sure that that this is still a 24bit jump */
119 if (!is_bl_op(op)) {
120 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
121 return -EINVAL;
122 }
123
124 /* lets find where the pointer goes */
125 tramp = find_bl_target(ip, op);
126
127 /*
128 * On PPC64 the trampoline looks like:
129 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
130 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
131 * Where the bytes 2,3,6 and 7 make up the 32bit offset
132 * to the TOC that holds the pointer.
133 * to jump to.
134 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
135 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
136 * The actually address is 32 bytes from the offset
137 * into the TOC.
138 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
139 */
140
141 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
142
143 /* Find where the trampoline jumps to */
144 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
145 printk(KERN_ERR "Failed to read %lx\n", tramp);
146 return -EFAULT;
147 }
148
149 pr_devel(" %08x %08x", jmp[0], jmp[1]);
150
151 /* verify that this is what we expect it to be */
152 if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
153 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
154 (jmp[2] != 0xf8410028) ||
155 (jmp[3] != 0xe96c0020) ||
156 (jmp[4] != 0xe84c0028)) {
157 printk(KERN_ERR "Not a trampoline\n");
158 return -EINVAL;
159 }
160
161 /* The bottom half is signed extended */
162 offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
163 (int)((short)jmp[1]);
164
165 pr_devel(" %x ", offset);
166
167 /* get the address this jumps too */
168 tramp = mod->arch.toc + offset + 32;
169 pr_devel("toc: %lx", tramp);
170
171 if (probe_kernel_read(jmp, (void *)tramp, 8)) {
172 printk(KERN_ERR "Failed to read %lx\n", tramp);
173 return -EFAULT;
174 }
175
176 pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
177
178#ifdef __LITTLE_ENDIAN__
179 ptr = ((unsigned long)jmp[1] << 32) + jmp[0];
180#else
181 ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
182#endif
183
184 /* This should match what was called */
185 if (ptr != ppc_function_entry((void *)addr)) {
186 printk(KERN_ERR "addr does not match %lx\n", ptr);
187 return -EINVAL;
188 }
189
190 /*
191 * We want to nop the line, but the next line is
192 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
193 * This needs to be turned to a nop too.
194 */
195 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
196 return -EFAULT;
197
198 if (op != 0xe8410028) {
199 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
200 return -EINVAL;
201 }
202
203 /*
204 * Milton Miller pointed out that we can not blindly do nops.
205 * If a task was preempted when calling a trace function,
206 * the nops will remove the way to restore the TOC in r2
207 * and the r2 TOC will get corrupted.
208 */
209
210 /*
211 * Replace:
212 * bl <tramp> <==== will be replaced with "b 1f"
213 * ld r2,40(r1)
214 * 1:
215 */
216 op = 0x48000008; /* b +8 */
217
218 if (patch_instruction((unsigned int *)ip, op))
219 return -EPERM;
220
221 return 0;
222}
223
224#else /* !PPC64 */
225static int
226__ftrace_make_nop(struct module *mod,
227 struct dyn_ftrace *rec, unsigned long addr)
228{
229 unsigned int op;
230 unsigned int jmp[4];
231 unsigned long ip = rec->ip;
232 unsigned long tramp;
233
234 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
235 return -EFAULT;
236
237 /* Make sure that that this is still a 24bit jump */
238 if (!is_bl_op(op)) {
239 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
240 return -EINVAL;
241 }
242
243 /* lets find where the pointer goes */
244 tramp = find_bl_target(ip, op);
245
246 /*
247 * On PPC32 the trampoline looks like:
248 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha
249 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l
250 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12
251 * 0x4e, 0x80, 0x04, 0x20 bctr
252 */
253
254 pr_devel("ip:%lx jumps to %lx", ip, tramp);
255
256 /* Find where the trampoline jumps to */
257 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
258 printk(KERN_ERR "Failed to read %lx\n", tramp);
259 return -EFAULT;
260 }
261
262 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
263
264 /* verify that this is what we expect it to be */
265 if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
266 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
267 (jmp[2] != 0x7d8903a6) ||
268 (jmp[3] != 0x4e800420)) {
269 printk(KERN_ERR "Not a trampoline\n");
270 return -EINVAL;
271 }
272
273 tramp = (jmp[1] & 0xffff) |
274 ((jmp[0] & 0xffff) << 16);
275 if (tramp & 0x8000)
276 tramp -= 0x10000;
277
278 pr_devel(" %lx ", tramp);
279
280 if (tramp != addr) {
281 printk(KERN_ERR
282 "Trampoline location %08lx does not match addr\n",
283 tramp);
284 return -EINVAL;
285 }
286
287 op = PPC_INST_NOP;
288
289 if (patch_instruction((unsigned int *)ip, op))
290 return -EPERM;
291
292 return 0;
293}
294#endif /* PPC64 */
295#endif /* CONFIG_MODULES */
296
297int ftrace_make_nop(struct module *mod,
298 struct dyn_ftrace *rec, unsigned long addr)
299{
300 unsigned long ip = rec->ip;
301 unsigned int old, new;
302
303 /*
304 * If the calling address is more that 24 bits away,
305 * then we had to use a trampoline to make the call.
306 * Otherwise just update the call site.
307 */
308 if (test_24bit_addr(ip, addr)) {
309 /* within range */
310 old = ftrace_call_replace(ip, addr, 1);
311 new = PPC_INST_NOP;
312 return ftrace_modify_code(ip, old, new);
313 }
314
315#ifdef CONFIG_MODULES
316 /*
317 * Out of range jumps are called from modules.
318 * We should either already have a pointer to the module
319 * or it has been passed in.
320 */
321 if (!rec->arch.mod) {
322 if (!mod) {
323 printk(KERN_ERR "No module loaded addr=%lx\n",
324 addr);
325 return -EFAULT;
326 }
327 rec->arch.mod = mod;
328 } else if (mod) {
329 if (mod != rec->arch.mod) {
330 printk(KERN_ERR
331 "Record mod %p not equal to passed in mod %p\n",
332 rec->arch.mod, mod);
333 return -EINVAL;
334 }
335 /* nothing to do if mod == rec->arch.mod */
336 } else
337 mod = rec->arch.mod;
338
339 return __ftrace_make_nop(mod, rec, addr);
340#else
341 /* We should not get here without modules */
342 return -EINVAL;
343#endif /* CONFIG_MODULES */
344}
345
346#ifdef CONFIG_MODULES
347#ifdef CONFIG_PPC64
348static int
349__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
350{
351 unsigned int op[2];
352 unsigned long ip = rec->ip;
353
354 /* read where this goes */
355 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
356 return -EFAULT;
357
358 /*
359 * It should be pointing to two nops or
360 * b +8; ld r2,40(r1)
361 */
362 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
363 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
364 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
365 return -EINVAL;
366 }
367
368 /* If we never set up a trampoline to ftrace_caller, then bail */
369 if (!rec->arch.mod->arch.tramp) {
370 printk(KERN_ERR "No ftrace trampoline\n");
371 return -EINVAL;
372 }
373
374 /* create the branch to the trampoline */
375 op[0] = create_branch((unsigned int *)ip,
376 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
377 if (!op[0]) {
378 printk(KERN_ERR "REL24 out of range!\n");
379 return -EINVAL;
380 }
381
382 /* ld r2,40(r1) */
383 op[1] = 0xe8410028;
384
385 pr_devel("write to %lx\n", rec->ip);
386
387 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
388 return -EPERM;
389
390 flush_icache_range(ip, ip + 8);
391
392 return 0;
393}
394#else
395static int
396__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
397{
398 unsigned int op;
399 unsigned long ip = rec->ip;
400
401 /* read where this goes */
402 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
403 return -EFAULT;
404
405 /* It should be pointing to a nop */
406 if (op != PPC_INST_NOP) {
407 printk(KERN_ERR "Expected NOP but have %x\n", op);
408 return -EINVAL;
409 }
410
411 /* If we never set up a trampoline to ftrace_caller, then bail */
412 if (!rec->arch.mod->arch.tramp) {
413 printk(KERN_ERR "No ftrace trampoline\n");
414 return -EINVAL;
415 }
416
417 /* create the branch to the trampoline */
418 op = create_branch((unsigned int *)ip,
419 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
420 if (!op) {
421 printk(KERN_ERR "REL24 out of range!\n");
422 return -EINVAL;
423 }
424
425 pr_devel("write to %lx\n", rec->ip);
426
427 if (patch_instruction((unsigned int *)ip, op))
428 return -EPERM;
429
430 return 0;
431}
432#endif /* CONFIG_PPC64 */
433#endif /* CONFIG_MODULES */
434
435int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
436{
437 unsigned long ip = rec->ip;
438 unsigned int old, new;
439
440 /*
441 * If the calling address is more that 24 bits away,
442 * then we had to use a trampoline to make the call.
443 * Otherwise just update the call site.
444 */
445 if (test_24bit_addr(ip, addr)) {
446 /* within range */
447 old = PPC_INST_NOP;
448 new = ftrace_call_replace(ip, addr, 1);
449 return ftrace_modify_code(ip, old, new);
450 }
451
452#ifdef CONFIG_MODULES
453 /*
454 * Out of range jumps are called from modules.
455 * Being that we are converting from nop, it had better
456 * already have a module defined.
457 */
458 if (!rec->arch.mod) {
459 printk(KERN_ERR "No module loaded\n");
460 return -EINVAL;
461 }
462
463 return __ftrace_make_call(rec, addr);
464#else
465 /* We should not get here without modules */
466 return -EINVAL;
467#endif /* CONFIG_MODULES */
468}
469
470int ftrace_update_ftrace_func(ftrace_func_t func)
471{
472 unsigned long ip = (unsigned long)(&ftrace_call);
473 unsigned int old, new;
474 int ret;
475
476 old = *(unsigned int *)&ftrace_call;
477 new = ftrace_call_replace(ip, (unsigned long)func, 1);
478 ret = ftrace_modify_code(ip, old, new);
479
480 return ret;
481}
482
483static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
484{
485 unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
486 int ret;
487
488 ret = ftrace_update_record(rec, enable);
489
490 switch (ret) {
491 case FTRACE_UPDATE_IGNORE:
492 return 0;
493 case FTRACE_UPDATE_MAKE_CALL:
494 return ftrace_make_call(rec, ftrace_addr);
495 case FTRACE_UPDATE_MAKE_NOP:
496 return ftrace_make_nop(NULL, rec, ftrace_addr);
497 }
498
499 return 0;
500}
501
502void ftrace_replace_code(int enable)
503{
504 struct ftrace_rec_iter *iter;
505 struct dyn_ftrace *rec;
506 int ret;
507
508 for (iter = ftrace_rec_iter_start(); iter;
509 iter = ftrace_rec_iter_next(iter)) {
510 rec = ftrace_rec_iter_record(iter);
511 ret = __ftrace_replace_code(rec, enable);
512 if (ret) {
513 ftrace_bug(ret, rec->ip);
514 return;
515 }
516 }
517}
518
519void arch_ftrace_update_code(int command)
520{
521 if (command & FTRACE_UPDATE_CALLS)
522 ftrace_replace_code(1);
523 else if (command & FTRACE_DISABLE_CALLS)
524 ftrace_replace_code(0);
525
526 if (command & FTRACE_UPDATE_TRACE_FUNC)
527 ftrace_update_ftrace_func(ftrace_trace_function);
528
529 if (command & FTRACE_START_FUNC_RET)
530 ftrace_enable_ftrace_graph_caller();
531 else if (command & FTRACE_STOP_FUNC_RET)
532 ftrace_disable_ftrace_graph_caller();
533}
534
535int __init ftrace_dyn_arch_init(void)
536{
537 return 0;
538}
539#endif /* CONFIG_DYNAMIC_FTRACE */
540
541#ifdef CONFIG_FUNCTION_GRAPH_TRACER
542
543#ifdef CONFIG_DYNAMIC_FTRACE
544extern void ftrace_graph_call(void);
545extern void ftrace_graph_stub(void);
546
547int ftrace_enable_ftrace_graph_caller(void)
548{
549 unsigned long ip = (unsigned long)(&ftrace_graph_call);
550 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
551 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
552 unsigned int old, new;
553
554 old = ftrace_call_replace(ip, stub, 0);
555 new = ftrace_call_replace(ip, addr, 0);
556
557 return ftrace_modify_code(ip, old, new);
558}
559
560int ftrace_disable_ftrace_graph_caller(void)
561{
562 unsigned long ip = (unsigned long)(&ftrace_graph_call);
563 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
564 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
565 unsigned int old, new;
566
567 old = ftrace_call_replace(ip, addr, 0);
568 new = ftrace_call_replace(ip, stub, 0);
569
570 return ftrace_modify_code(ip, old, new);
571}
572#endif /* CONFIG_DYNAMIC_FTRACE */
573
574#ifdef CONFIG_PPC64
575extern void mod_return_to_handler(void);
576#endif
577
578/*
579 * Hook the return address and push it in the stack of return addrs
580 * in current thread info.
581 */
582void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
583{
584 unsigned long old;
585 int faulted;
586 struct ftrace_graph_ent trace;
587 unsigned long return_hooker = (unsigned long)&return_to_handler;
588
589 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
590 return;
591
592#ifdef CONFIG_PPC64
593 /* non core kernel code needs to save and restore the TOC */
594 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
595 return_hooker = (unsigned long)&mod_return_to_handler;
596#endif
597
598 return_hooker = ppc_function_entry((void *)return_hooker);
599
600 /*
601 * Protect against fault, even if it shouldn't
602 * happen. This tool is too much intrusive to
603 * ignore such a protection.
604 */
605 asm volatile(
606 "1: " PPC_LL "%[old], 0(%[parent])\n"
607 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
608 " li %[faulted], 0\n"
609 "3:\n"
610
611 ".section .fixup, \"ax\"\n"
612 "4: li %[faulted], 1\n"
613 " b 3b\n"
614 ".previous\n"
615
616 ".section __ex_table,\"a\"\n"
617 PPC_LONG_ALIGN "\n"
618 PPC_LONG "1b,4b\n"
619 PPC_LONG "2b,4b\n"
620 ".previous"
621
622 : [old] "=&r" (old), [faulted] "=r" (faulted)
623 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
624 : "memory"
625 );
626
627 if (unlikely(faulted)) {
628 ftrace_graph_stop();
629 WARN_ON(1);
630 return;
631 }
632
633 trace.func = self_addr;
634 trace.depth = current->curr_ret_stack + 1;
635
636 /* Only trace if the calling function expects to */
637 if (!ftrace_graph_entry(&trace)) {
638 *parent = old;
639 return;
640 }
641
642 if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY)
643 *parent = old;
644}
645#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
646
647#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
648unsigned long __init arch_syscall_addr(int nr)
649{
650 return sys_call_table[nr*2];
651}
652#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */