Loading...
Note: File does not exist in v5.9.
1/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
7 *
8 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
10 *
11 */
12
13#define pr_fmt(fmt) "ftrace-powerpc: " fmt
14
15#include <linux/spinlock.h>
16#include <linux/hardirq.h>
17#include <linux/uaccess.h>
18#include <linux/module.h>
19#include <linux/ftrace.h>
20#include <linux/percpu.h>
21#include <linux/init.h>
22#include <linux/list.h>
23
24#include <asm/cacheflush.h>
25#include <asm/code-patching.h>
26#include <asm/ftrace.h>
27#include <asm/syscall.h>
28
29
30#ifdef CONFIG_DYNAMIC_FTRACE
31static unsigned int
32ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
33{
34 unsigned int op;
35
36 addr = ppc_function_entry((void *)addr);
37
38 /* if (link) set op to 'bl' else 'b' */
39 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
40
41 return op;
42}
43
44static int
45ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
46{
47 unsigned int replaced;
48
49 /*
50 * Note:
51 * We are paranoid about modifying text, as if a bug was to happen, it
52 * could cause us to read or write to someplace that could cause harm.
53 * Carefully read and modify the code with probe_kernel_*(), and make
54 * sure what we read is what we expected it to be before modifying it.
55 */
56
57 /* read the text we want to modify */
58 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
59 return -EFAULT;
60
61 /* Make sure it is what we expect it to be */
62 if (replaced != old) {
63 pr_err("%p: replaced (%#x) != old (%#x)",
64 (void *)ip, replaced, old);
65 return -EINVAL;
66 }
67
68 /* replace the text with the new text */
69 if (patch_instruction((unsigned int *)ip, new))
70 return -EPERM;
71
72 return 0;
73}
74
75/*
76 * Helper functions that are the same for both PPC64 and PPC32.
77 */
78static int test_24bit_addr(unsigned long ip, unsigned long addr)
79{
80 addr = ppc_function_entry((void *)addr);
81
82 /* use the create_branch to verify that this offset can be branched */
83 return create_branch((unsigned int *)ip, addr, 0);
84}
85
86#ifdef CONFIG_MODULES
87
88static int is_bl_op(unsigned int op)
89{
90 return (op & 0xfc000003) == 0x48000001;
91}
92
93static unsigned long find_bl_target(unsigned long ip, unsigned int op)
94{
95 static int offset;
96
97 offset = (op & 0x03fffffc);
98 /* make it signed */
99 if (offset & 0x02000000)
100 offset |= 0xfe000000;
101
102 return ip + (long)offset;
103}
104
105#ifdef CONFIG_PPC64
106static int
107__ftrace_make_nop(struct module *mod,
108 struct dyn_ftrace *rec, unsigned long addr)
109{
110 unsigned long entry, ptr, tramp;
111 unsigned long ip = rec->ip;
112 unsigned int op, pop;
113
114 /* read where this goes */
115 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
116 pr_err("Fetching opcode failed.\n");
117 return -EFAULT;
118 }
119
120 /* Make sure that that this is still a 24bit jump */
121 if (!is_bl_op(op)) {
122 pr_err("Not expected bl: opcode is %x\n", op);
123 return -EINVAL;
124 }
125
126 /* lets find where the pointer goes */
127 tramp = find_bl_target(ip, op);
128
129 pr_devel("ip:%lx jumps to %lx", ip, tramp);
130
131 if (module_trampoline_target(mod, tramp, &ptr)) {
132 pr_err("Failed to get trampoline target\n");
133 return -EFAULT;
134 }
135
136 pr_devel("trampoline target %lx", ptr);
137
138 entry = ppc_global_function_entry((void *)addr);
139 /* This should match what was called */
140 if (ptr != entry) {
141 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
142 return -EINVAL;
143 }
144
145#ifdef CC_USING_MPROFILE_KERNEL
146 /* When using -mkernel_profile there is no load to jump over */
147 pop = PPC_INST_NOP;
148
149 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
150 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
151 return -EFAULT;
152 }
153
154 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
155 if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
156 pr_err("Unexpected instruction %08x around bl _mcount\n", op);
157 return -EINVAL;
158 }
159#else
160 /*
161 * Our original call site looks like:
162 *
163 * bl <tramp>
164 * ld r2,XX(r1)
165 *
166 * Milton Miller pointed out that we can not simply nop the branch.
167 * If a task was preempted when calling a trace function, the nops
168 * will remove the way to restore the TOC in r2 and the r2 TOC will
169 * get corrupted.
170 *
171 * Use a b +8 to jump over the load.
172 */
173
174 pop = PPC_INST_BRANCH | 8; /* b +8 */
175
176 /*
177 * Check what is in the next instruction. We can see ld r2,40(r1), but
178 * on first pass after boot we will see mflr r0.
179 */
180 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
181 pr_err("Fetching op failed.\n");
182 return -EFAULT;
183 }
184
185 if (op != PPC_INST_LD_TOC) {
186 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
187 return -EINVAL;
188 }
189#endif /* CC_USING_MPROFILE_KERNEL */
190
191 if (patch_instruction((unsigned int *)ip, pop)) {
192 pr_err("Patching NOP failed.\n");
193 return -EPERM;
194 }
195
196 return 0;
197}
198
199#else /* !PPC64 */
200static int
201__ftrace_make_nop(struct module *mod,
202 struct dyn_ftrace *rec, unsigned long addr)
203{
204 unsigned int op;
205 unsigned int jmp[4];
206 unsigned long ip = rec->ip;
207 unsigned long tramp;
208
209 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
210 return -EFAULT;
211
212 /* Make sure that that this is still a 24bit jump */
213 if (!is_bl_op(op)) {
214 pr_err("Not expected bl: opcode is %x\n", op);
215 return -EINVAL;
216 }
217
218 /* lets find where the pointer goes */
219 tramp = find_bl_target(ip, op);
220
221 /*
222 * On PPC32 the trampoline looks like:
223 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha
224 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l
225 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12
226 * 0x4e, 0x80, 0x04, 0x20 bctr
227 */
228
229 pr_devel("ip:%lx jumps to %lx", ip, tramp);
230
231 /* Find where the trampoline jumps to */
232 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
233 pr_err("Failed to read %lx\n", tramp);
234 return -EFAULT;
235 }
236
237 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
238
239 /* verify that this is what we expect it to be */
240 if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
241 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
242 (jmp[2] != 0x7d8903a6) ||
243 (jmp[3] != 0x4e800420)) {
244 pr_err("Not a trampoline\n");
245 return -EINVAL;
246 }
247
248 tramp = (jmp[1] & 0xffff) |
249 ((jmp[0] & 0xffff) << 16);
250 if (tramp & 0x8000)
251 tramp -= 0x10000;
252
253 pr_devel(" %lx ", tramp);
254
255 if (tramp != addr) {
256 pr_err("Trampoline location %08lx does not match addr\n",
257 tramp);
258 return -EINVAL;
259 }
260
261 op = PPC_INST_NOP;
262
263 if (patch_instruction((unsigned int *)ip, op))
264 return -EPERM;
265
266 return 0;
267}
268#endif /* PPC64 */
269#endif /* CONFIG_MODULES */
270
271int ftrace_make_nop(struct module *mod,
272 struct dyn_ftrace *rec, unsigned long addr)
273{
274 unsigned long ip = rec->ip;
275 unsigned int old, new;
276
277 /*
278 * If the calling address is more that 24 bits away,
279 * then we had to use a trampoline to make the call.
280 * Otherwise just update the call site.
281 */
282 if (test_24bit_addr(ip, addr)) {
283 /* within range */
284 old = ftrace_call_replace(ip, addr, 1);
285 new = PPC_INST_NOP;
286 return ftrace_modify_code(ip, old, new);
287 }
288
289#ifdef CONFIG_MODULES
290 /*
291 * Out of range jumps are called from modules.
292 * We should either already have a pointer to the module
293 * or it has been passed in.
294 */
295 if (!rec->arch.mod) {
296 if (!mod) {
297 pr_err("No module loaded addr=%lx\n", addr);
298 return -EFAULT;
299 }
300 rec->arch.mod = mod;
301 } else if (mod) {
302 if (mod != rec->arch.mod) {
303 pr_err("Record mod %p not equal to passed in mod %p\n",
304 rec->arch.mod, mod);
305 return -EINVAL;
306 }
307 /* nothing to do if mod == rec->arch.mod */
308 } else
309 mod = rec->arch.mod;
310
311 return __ftrace_make_nop(mod, rec, addr);
312#else
313 /* We should not get here without modules */
314 return -EINVAL;
315#endif /* CONFIG_MODULES */
316}
317
318#ifdef CONFIG_MODULES
319#ifdef CONFIG_PPC64
320/*
321 * Examine the existing instructions for __ftrace_make_call.
322 * They should effectively be a NOP, and follow formal constraints,
323 * depending on the ABI. Return false if they don't.
324 */
325#ifndef CC_USING_MPROFILE_KERNEL
326static int
327expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
328{
329 /*
330 * We expect to see:
331 *
332 * b +8
333 * ld r2,XX(r1)
334 *
335 * The load offset is different depending on the ABI. For simplicity
336 * just mask it out when doing the compare.
337 */
338 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
339 return 0;
340 return 1;
341}
342#else
343static int
344expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
345{
346 /* look for patched "NOP" on ppc64 with -mprofile-kernel */
347 if (op0 != PPC_INST_NOP)
348 return 0;
349 return 1;
350}
351#endif
352
353static int
354__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
355{
356 unsigned int op[2];
357 void *ip = (void *)rec->ip;
358
359 /* read where this goes */
360 if (probe_kernel_read(op, ip, sizeof(op)))
361 return -EFAULT;
362
363 if (!expected_nop_sequence(ip, op[0], op[1])) {
364 pr_err("Unexpected call sequence at %p: %x %x\n",
365 ip, op[0], op[1]);
366 return -EINVAL;
367 }
368
369 /* If we never set up a trampoline to ftrace_caller, then bail */
370 if (!rec->arch.mod->arch.tramp) {
371 pr_err("No ftrace trampoline\n");
372 return -EINVAL;
373 }
374
375 /* Ensure branch is within 24 bits */
376 if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
377 pr_err("Branch out of range\n");
378 return -EINVAL;
379 }
380
381 if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
382 pr_err("REL24 out of range!\n");
383 return -EINVAL;
384 }
385
386 return 0;
387}
388
389#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
390int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
391 unsigned long addr)
392{
393 return ftrace_make_call(rec, addr);
394}
395#endif
396
397#else /* !CONFIG_PPC64: */
398static int
399__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
400{
401 unsigned int op;
402 unsigned long ip = rec->ip;
403
404 /* read where this goes */
405 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
406 return -EFAULT;
407
408 /* It should be pointing to a nop */
409 if (op != PPC_INST_NOP) {
410 pr_err("Expected NOP but have %x\n", op);
411 return -EINVAL;
412 }
413
414 /* If we never set up a trampoline to ftrace_caller, then bail */
415 if (!rec->arch.mod->arch.tramp) {
416 pr_err("No ftrace trampoline\n");
417 return -EINVAL;
418 }
419
420 /* create the branch to the trampoline */
421 op = create_branch((unsigned int *)ip,
422 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
423 if (!op) {
424 pr_err("REL24 out of range!\n");
425 return -EINVAL;
426 }
427
428 pr_devel("write to %lx\n", rec->ip);
429
430 if (patch_instruction((unsigned int *)ip, op))
431 return -EPERM;
432
433 return 0;
434}
435#endif /* CONFIG_PPC64 */
436#endif /* CONFIG_MODULES */
437
438int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
439{
440 unsigned long ip = rec->ip;
441 unsigned int old, new;
442
443 /*
444 * If the calling address is more that 24 bits away,
445 * then we had to use a trampoline to make the call.
446 * Otherwise just update the call site.
447 */
448 if (test_24bit_addr(ip, addr)) {
449 /* within range */
450 old = PPC_INST_NOP;
451 new = ftrace_call_replace(ip, addr, 1);
452 return ftrace_modify_code(ip, old, new);
453 }
454
455#ifdef CONFIG_MODULES
456 /*
457 * Out of range jumps are called from modules.
458 * Being that we are converting from nop, it had better
459 * already have a module defined.
460 */
461 if (!rec->arch.mod) {
462 pr_err("No module loaded\n");
463 return -EINVAL;
464 }
465
466 return __ftrace_make_call(rec, addr);
467#else
468 /* We should not get here without modules */
469 return -EINVAL;
470#endif /* CONFIG_MODULES */
471}
472
473int ftrace_update_ftrace_func(ftrace_func_t func)
474{
475 unsigned long ip = (unsigned long)(&ftrace_call);
476 unsigned int old, new;
477 int ret;
478
479 old = *(unsigned int *)&ftrace_call;
480 new = ftrace_call_replace(ip, (unsigned long)func, 1);
481 ret = ftrace_modify_code(ip, old, new);
482
483 return ret;
484}
485
486static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
487{
488 unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
489 int ret;
490
491 ret = ftrace_update_record(rec, enable);
492
493 switch (ret) {
494 case FTRACE_UPDATE_IGNORE:
495 return 0;
496 case FTRACE_UPDATE_MAKE_CALL:
497 return ftrace_make_call(rec, ftrace_addr);
498 case FTRACE_UPDATE_MAKE_NOP:
499 return ftrace_make_nop(NULL, rec, ftrace_addr);
500 }
501
502 return 0;
503}
504
505void ftrace_replace_code(int enable)
506{
507 struct ftrace_rec_iter *iter;
508 struct dyn_ftrace *rec;
509 int ret;
510
511 for (iter = ftrace_rec_iter_start(); iter;
512 iter = ftrace_rec_iter_next(iter)) {
513 rec = ftrace_rec_iter_record(iter);
514 ret = __ftrace_replace_code(rec, enable);
515 if (ret) {
516 ftrace_bug(ret, rec);
517 return;
518 }
519 }
520}
521
522/*
523 * Use the default ftrace_modify_all_code, but without
524 * stop_machine().
525 */
526void arch_ftrace_update_code(int command)
527{
528 ftrace_modify_all_code(command);
529}
530
531int __init ftrace_dyn_arch_init(void)
532{
533 return 0;
534}
535#endif /* CONFIG_DYNAMIC_FTRACE */
536
537#ifdef CONFIG_FUNCTION_GRAPH_TRACER
538
539#ifdef CONFIG_DYNAMIC_FTRACE
540extern void ftrace_graph_call(void);
541extern void ftrace_graph_stub(void);
542
543int ftrace_enable_ftrace_graph_caller(void)
544{
545 unsigned long ip = (unsigned long)(&ftrace_graph_call);
546 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
547 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
548 unsigned int old, new;
549
550 old = ftrace_call_replace(ip, stub, 0);
551 new = ftrace_call_replace(ip, addr, 0);
552
553 return ftrace_modify_code(ip, old, new);
554}
555
556int ftrace_disable_ftrace_graph_caller(void)
557{
558 unsigned long ip = (unsigned long)(&ftrace_graph_call);
559 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
560 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
561 unsigned int old, new;
562
563 old = ftrace_call_replace(ip, addr, 0);
564 new = ftrace_call_replace(ip, stub, 0);
565
566 return ftrace_modify_code(ip, old, new);
567}
568#endif /* CONFIG_DYNAMIC_FTRACE */
569
570/*
571 * Hook the return address and push it in the stack of return addrs
572 * in current thread info. Return the address we want to divert to.
573 */
574unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
575{
576 struct ftrace_graph_ent trace;
577 unsigned long return_hooker;
578
579 if (unlikely(ftrace_graph_is_dead()))
580 goto out;
581
582 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
583 goto out;
584
585 return_hooker = ppc_function_entry(return_to_handler);
586
587 trace.func = ip;
588 trace.depth = current->curr_ret_stack + 1;
589
590 /* Only trace if the calling function expects to */
591 if (!ftrace_graph_entry(&trace))
592 goto out;
593
594 if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
595 NULL) == -EBUSY)
596 goto out;
597
598 parent = return_hooker;
599out:
600 return parent;
601}
602#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
603
604#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
605unsigned long __init arch_syscall_addr(int nr)
606{
607 return sys_call_table[nr*2];
608}
609#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
610
611#ifdef PPC64_ELF_ABI_v1
612char *arch_ftrace_match_adjust(char *str, const char *search)
613{
614 if (str[0] == '.' && search[0] != '.')
615 return str + 1;
616 else
617 return str;
618}
619#endif /* PPC64_ELF_ABI_v1 */