Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 *
8 * Highly modified by Steven Rostedt (VMware).
9 */
10#include <linux/suspend.h>
11#include <linux/ftrace.h>
12#include <linux/slab.h>
13
14#include <trace/events/sched.h>
15
16#include "ftrace_internal.h"
17
18#ifdef CONFIG_DYNAMIC_FTRACE
19#define ASSIGN_OPS_HASH(opsname, val) \
20 .func_hash = val, \
21 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
22#else
23#define ASSIGN_OPS_HASH(opsname, val)
24#endif
25
26static bool kill_ftrace_graph;
27int ftrace_graph_active;
28
29/* Both enabled by default (can be cleared by function_graph tracer flags */
30static bool fgraph_sleep_time = true;
31
32/**
33 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
34 *
35 * ftrace_graph_stop() is called when a severe error is detected in
36 * the function graph tracing. This function is called by the critical
37 * paths of function graph to keep those paths from doing any more harm.
38 */
39bool ftrace_graph_is_dead(void)
40{
41 return kill_ftrace_graph;
42}
43
44/**
45 * ftrace_graph_stop - set to permanently disable function graph tracincg
46 *
47 * In case of an error int function graph tracing, this is called
48 * to try to keep function graph tracing from causing any more harm.
49 * Usually this is pretty severe and this is called to try to at least
50 * get a warning out to the user.
51 */
52void ftrace_graph_stop(void)
53{
54 kill_ftrace_graph = true;
55}
56
57/* Add a function return address to the trace stack on thread info.*/
58static int
59ftrace_push_return_trace(unsigned long ret, unsigned long func,
60 unsigned long frame_pointer, unsigned long *retp)
61{
62 unsigned long long calltime;
63 int index;
64
65 if (unlikely(ftrace_graph_is_dead()))
66 return -EBUSY;
67
68 if (!current->ret_stack)
69 return -EBUSY;
70
71 /*
72 * We must make sure the ret_stack is tested before we read
73 * anything else.
74 */
75 smp_rmb();
76
77 /* The return trace stack is full */
78 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
79 atomic_inc(¤t->trace_overrun);
80 return -EBUSY;
81 }
82
83 calltime = trace_clock_local();
84
85 index = ++current->curr_ret_stack;
86 barrier();
87 current->ret_stack[index].ret = ret;
88 current->ret_stack[index].func = func;
89 current->ret_stack[index].calltime = calltime;
90#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
91 current->ret_stack[index].fp = frame_pointer;
92#endif
93#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
94 current->ret_stack[index].retp = retp;
95#endif
96 return 0;
97}
98
99/*
100 * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct
101 * functions. But those archs currently don't support direct functions
102 * anyway, and ftrace_find_rec_direct() is just a stub for them.
103 * Define MCOUNT_INSN_SIZE to keep those archs compiling.
104 */
105#ifndef MCOUNT_INSN_SIZE
106/* Make sure this only works without direct calls */
107# ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
108# error MCOUNT_INSN_SIZE not defined with direct calls enabled
109# endif
110# define MCOUNT_INSN_SIZE 0
111#endif
112
113int function_graph_enter(unsigned long ret, unsigned long func,
114 unsigned long frame_pointer, unsigned long *retp)
115{
116 struct ftrace_graph_ent trace;
117
118 /*
119 * Skip graph tracing if the return location is served by direct trampoline,
120 * since call sequence and return addresses is unpredicatable anymore.
121 * Ex: BPF trampoline may call original function and may skip frame
122 * depending on type of BPF programs attached.
123 */
124 if (ftrace_direct_func_count &&
125 ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))
126 return -EBUSY;
127 trace.func = func;
128 trace.depth = ++current->curr_ret_depth;
129
130 if (ftrace_push_return_trace(ret, func, frame_pointer, retp))
131 goto out;
132
133 /* Only trace if the calling function expects to */
134 if (!ftrace_graph_entry(&trace))
135 goto out_ret;
136
137 return 0;
138 out_ret:
139 current->curr_ret_stack--;
140 out:
141 current->curr_ret_depth--;
142 return -EBUSY;
143}
144
145/* Retrieve a function return address to the trace stack on thread info.*/
146static void
147ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
148 unsigned long frame_pointer)
149{
150 int index;
151
152 index = current->curr_ret_stack;
153
154 if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) {
155 ftrace_graph_stop();
156 WARN_ON(1);
157 /* Might as well panic, otherwise we have no where to go */
158 *ret = (unsigned long)panic;
159 return;
160 }
161
162#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
163 /*
164 * The arch may choose to record the frame pointer used
165 * and check it here to make sure that it is what we expect it
166 * to be. If gcc does not set the place holder of the return
167 * address in the frame pointer, and does a copy instead, then
168 * the function graph trace will fail. This test detects this
169 * case.
170 *
171 * Currently, x86_32 with optimize for size (-Os) makes the latest
172 * gcc do the above.
173 *
174 * Note, -mfentry does not use frame pointers, and this test
175 * is not needed if CC_USING_FENTRY is set.
176 */
177 if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
178 ftrace_graph_stop();
179 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
180 " from func %ps return to %lx\n",
181 current->ret_stack[index].fp,
182 frame_pointer,
183 (void *)current->ret_stack[index].func,
184 current->ret_stack[index].ret);
185 *ret = (unsigned long)panic;
186 return;
187 }
188#endif
189
190 *ret = current->ret_stack[index].ret;
191 trace->func = current->ret_stack[index].func;
192 trace->calltime = current->ret_stack[index].calltime;
193 trace->overrun = atomic_read(¤t->trace_overrun);
194 trace->depth = current->curr_ret_depth--;
195 /*
196 * We still want to trace interrupts coming in if
197 * max_depth is set to 1. Make sure the decrement is
198 * seen before ftrace_graph_return.
199 */
200 barrier();
201}
202
203/*
204 * Hibernation protection.
205 * The state of the current task is too much unstable during
206 * suspend/restore to disk. We want to protect against that.
207 */
208static int
209ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
210 void *unused)
211{
212 switch (state) {
213 case PM_HIBERNATION_PREPARE:
214 pause_graph_tracing();
215 break;
216
217 case PM_POST_HIBERNATION:
218 unpause_graph_tracing();
219 break;
220 }
221 return NOTIFY_DONE;
222}
223
224static struct notifier_block ftrace_suspend_notifier = {
225 .notifier_call = ftrace_suspend_notifier_call,
226};
227
228/*
229 * Send the trace to the ring-buffer.
230 * @return the original return address.
231 */
232unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
233{
234 struct ftrace_graph_ret trace;
235 unsigned long ret;
236
237 ftrace_pop_return_trace(&trace, &ret, frame_pointer);
238 trace.rettime = trace_clock_local();
239 ftrace_graph_return(&trace);
240 /*
241 * The ftrace_graph_return() may still access the current
242 * ret_stack structure, we need to make sure the update of
243 * curr_ret_stack is after that.
244 */
245 barrier();
246 current->curr_ret_stack--;
247
248 if (unlikely(!ret)) {
249 ftrace_graph_stop();
250 WARN_ON(1);
251 /* Might as well panic. What else to do? */
252 ret = (unsigned long)panic;
253 }
254
255 return ret;
256}
257
258/**
259 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
260 * @task: The task to read the shadow stack from
261 * @idx: Index down the shadow stack
262 *
263 * Return the ret_struct on the shadow stack of the @task at the
264 * call graph at @idx starting with zero. If @idx is zero, it
265 * will return the last saved ret_stack entry. If it is greater than
266 * zero, it will return the corresponding ret_stack for the depth
267 * of saved return addresses.
268 */
269struct ftrace_ret_stack *
270ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
271{
272 idx = task->curr_ret_stack - idx;
273
274 if (idx >= 0 && idx <= task->curr_ret_stack)
275 return &task->ret_stack[idx];
276
277 return NULL;
278}
279
280/**
281 * ftrace_graph_ret_addr - convert a potentially modified stack return address
282 * to its original value
283 *
284 * This function can be called by stack unwinding code to convert a found stack
285 * return address ('ret') to its original value, in case the function graph
286 * tracer has modified it to be 'return_to_handler'. If the address hasn't
287 * been modified, the unchanged value of 'ret' is returned.
288 *
289 * 'idx' is a state variable which should be initialized by the caller to zero
290 * before the first call.
291 *
292 * 'retp' is a pointer to the return address on the stack. It's ignored if
293 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
294 */
295#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
296unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
297 unsigned long ret, unsigned long *retp)
298{
299 int index = task->curr_ret_stack;
300 int i;
301
302 if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler))
303 return ret;
304
305 if (index < 0)
306 return ret;
307
308 for (i = 0; i <= index; i++)
309 if (task->ret_stack[i].retp == retp)
310 return task->ret_stack[i].ret;
311
312 return ret;
313}
314#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
315unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
316 unsigned long ret, unsigned long *retp)
317{
318 int task_idx;
319
320 if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler))
321 return ret;
322
323 task_idx = task->curr_ret_stack;
324
325 if (!task->ret_stack || task_idx < *idx)
326 return ret;
327
328 task_idx -= *idx;
329 (*idx)++;
330
331 return task->ret_stack[task_idx].ret;
332}
333#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
334
335static struct ftrace_ops graph_ops = {
336 .func = ftrace_stub,
337 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
338 FTRACE_OPS_FL_INITIALIZED |
339 FTRACE_OPS_FL_PID |
340 FTRACE_OPS_FL_STUB,
341#ifdef FTRACE_GRAPH_TRAMP_ADDR
342 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
343 /* trampoline_size is only needed for dynamically allocated tramps */
344#endif
345 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
346};
347
348void ftrace_graph_sleep_time_control(bool enable)
349{
350 fgraph_sleep_time = enable;
351}
352
353int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
354{
355 return 0;
356}
357
358/*
359 * Simply points to ftrace_stub, but with the proper protocol.
360 * Defined by the linker script in linux/vmlinux.lds.h
361 */
362extern void ftrace_stub_graph(struct ftrace_graph_ret *);
363
364/* The callbacks that hook a function */
365trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph;
366trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
367static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
368
369/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
370static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
371{
372 int i;
373 int ret = 0;
374 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
375 struct task_struct *g, *t;
376
377 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
378 ret_stack_list[i] =
379 kmalloc_array(FTRACE_RETFUNC_DEPTH,
380 sizeof(struct ftrace_ret_stack),
381 GFP_KERNEL);
382 if (!ret_stack_list[i]) {
383 start = 0;
384 end = i;
385 ret = -ENOMEM;
386 goto free;
387 }
388 }
389
390 read_lock(&tasklist_lock);
391 do_each_thread(g, t) {
392 if (start == end) {
393 ret = -EAGAIN;
394 goto unlock;
395 }
396
397 if (t->ret_stack == NULL) {
398 atomic_set(&t->tracing_graph_pause, 0);
399 atomic_set(&t->trace_overrun, 0);
400 t->curr_ret_stack = -1;
401 t->curr_ret_depth = -1;
402 /* Make sure the tasks see the -1 first: */
403 smp_wmb();
404 t->ret_stack = ret_stack_list[start++];
405 }
406 } while_each_thread(g, t);
407
408unlock:
409 read_unlock(&tasklist_lock);
410free:
411 for (i = start; i < end; i++)
412 kfree(ret_stack_list[i]);
413 return ret;
414}
415
416static void
417ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
418 struct task_struct *prev, struct task_struct *next)
419{
420 unsigned long long timestamp;
421 int index;
422
423 /*
424 * Does the user want to count the time a function was asleep.
425 * If so, do not update the time stamps.
426 */
427 if (fgraph_sleep_time)
428 return;
429
430 timestamp = trace_clock_local();
431
432 prev->ftrace_timestamp = timestamp;
433
434 /* only process tasks that we timestamped */
435 if (!next->ftrace_timestamp)
436 return;
437
438 /*
439 * Update all the counters in next to make up for the
440 * time next was sleeping.
441 */
442 timestamp -= next->ftrace_timestamp;
443
444 for (index = next->curr_ret_stack; index >= 0; index--)
445 next->ret_stack[index].calltime += timestamp;
446}
447
448static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
449{
450 if (!ftrace_ops_test(&global_ops, trace->func, NULL))
451 return 0;
452 return __ftrace_graph_entry(trace);
453}
454
455/*
456 * The function graph tracer should only trace the functions defined
457 * by set_ftrace_filter and set_ftrace_notrace. If another function
458 * tracer ops is registered, the graph tracer requires testing the
459 * function against the global ops, and not just trace any function
460 * that any ftrace_ops registered.
461 */
462void update_function_graph_func(void)
463{
464 struct ftrace_ops *op;
465 bool do_test = false;
466
467 /*
468 * The graph and global ops share the same set of functions
469 * to test. If any other ops is on the list, then
470 * the graph tracing needs to test if its the function
471 * it should call.
472 */
473 do_for_each_ftrace_op(op, ftrace_ops_list) {
474 if (op != &global_ops && op != &graph_ops &&
475 op != &ftrace_list_end) {
476 do_test = true;
477 /* in double loop, break out with goto */
478 goto out;
479 }
480 } while_for_each_ftrace_op(op);
481 out:
482 if (do_test)
483 ftrace_graph_entry = ftrace_graph_entry_test;
484 else
485 ftrace_graph_entry = __ftrace_graph_entry;
486}
487
488static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
489
490static void
491graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
492{
493 atomic_set(&t->tracing_graph_pause, 0);
494 atomic_set(&t->trace_overrun, 0);
495 t->ftrace_timestamp = 0;
496 /* make curr_ret_stack visible before we add the ret_stack */
497 smp_wmb();
498 t->ret_stack = ret_stack;
499}
500
501/*
502 * Allocate a return stack for the idle task. May be the first
503 * time through, or it may be done by CPU hotplug online.
504 */
505void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
506{
507 t->curr_ret_stack = -1;
508 t->curr_ret_depth = -1;
509 /*
510 * The idle task has no parent, it either has its own
511 * stack or no stack at all.
512 */
513 if (t->ret_stack)
514 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
515
516 if (ftrace_graph_active) {
517 struct ftrace_ret_stack *ret_stack;
518
519 ret_stack = per_cpu(idle_ret_stack, cpu);
520 if (!ret_stack) {
521 ret_stack =
522 kmalloc_array(FTRACE_RETFUNC_DEPTH,
523 sizeof(struct ftrace_ret_stack),
524 GFP_KERNEL);
525 if (!ret_stack)
526 return;
527 per_cpu(idle_ret_stack, cpu) = ret_stack;
528 }
529 graph_init_task(t, ret_stack);
530 }
531}
532
533/* Allocate a return stack for newly created task */
534void ftrace_graph_init_task(struct task_struct *t)
535{
536 /* Make sure we do not use the parent ret_stack */
537 t->ret_stack = NULL;
538 t->curr_ret_stack = -1;
539 t->curr_ret_depth = -1;
540
541 if (ftrace_graph_active) {
542 struct ftrace_ret_stack *ret_stack;
543
544 ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
545 sizeof(struct ftrace_ret_stack),
546 GFP_KERNEL);
547 if (!ret_stack)
548 return;
549 graph_init_task(t, ret_stack);
550 }
551}
552
553void ftrace_graph_exit_task(struct task_struct *t)
554{
555 struct ftrace_ret_stack *ret_stack = t->ret_stack;
556
557 t->ret_stack = NULL;
558 /* NULL must become visible to IRQs before we free it: */
559 barrier();
560
561 kfree(ret_stack);
562}
563
564/* Allocate a return stack for each task */
565static int start_graph_tracing(void)
566{
567 struct ftrace_ret_stack **ret_stack_list;
568 int ret, cpu;
569
570 ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
571 sizeof(struct ftrace_ret_stack *),
572 GFP_KERNEL);
573
574 if (!ret_stack_list)
575 return -ENOMEM;
576
577 /* The cpu_boot init_task->ret_stack will never be freed */
578 for_each_online_cpu(cpu) {
579 if (!idle_task(cpu)->ret_stack)
580 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
581 }
582
583 do {
584 ret = alloc_retstack_tasklist(ret_stack_list);
585 } while (ret == -EAGAIN);
586
587 if (!ret) {
588 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
589 if (ret)
590 pr_info("ftrace_graph: Couldn't activate tracepoint"
591 " probe to kernel_sched_switch\n");
592 }
593
594 kfree(ret_stack_list);
595 return ret;
596}
597
598int register_ftrace_graph(struct fgraph_ops *gops)
599{
600 int ret = 0;
601
602 mutex_lock(&ftrace_lock);
603
604 /* we currently allow only one tracer registered at a time */
605 if (ftrace_graph_active) {
606 ret = -EBUSY;
607 goto out;
608 }
609
610 register_pm_notifier(&ftrace_suspend_notifier);
611
612 ftrace_graph_active++;
613 ret = start_graph_tracing();
614 if (ret) {
615 ftrace_graph_active--;
616 goto out;
617 }
618
619 ftrace_graph_return = gops->retfunc;
620
621 /*
622 * Update the indirect function to the entryfunc, and the
623 * function that gets called to the entry_test first. Then
624 * call the update fgraph entry function to determine if
625 * the entryfunc should be called directly or not.
626 */
627 __ftrace_graph_entry = gops->entryfunc;
628 ftrace_graph_entry = ftrace_graph_entry_test;
629 update_function_graph_func();
630
631 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
632out:
633 mutex_unlock(&ftrace_lock);
634 return ret;
635}
636
637void unregister_ftrace_graph(struct fgraph_ops *gops)
638{
639 mutex_lock(&ftrace_lock);
640
641 if (unlikely(!ftrace_graph_active))
642 goto out;
643
644 ftrace_graph_active--;
645 ftrace_graph_return = ftrace_stub_graph;
646 ftrace_graph_entry = ftrace_graph_entry_stub;
647 __ftrace_graph_entry = ftrace_graph_entry_stub;
648 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
649 unregister_pm_notifier(&ftrace_suspend_notifier);
650 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
651
652 out:
653 mutex_unlock(&ftrace_lock);
654}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 *
8 * Highly modified by Steven Rostedt (VMware).
9 */
10#include <linux/jump_label.h>
11#include <linux/suspend.h>
12#include <linux/ftrace.h>
13#include <linux/slab.h>
14
15#include <trace/events/sched.h>
16
17#include "ftrace_internal.h"
18#include "trace.h"
19
20#ifdef CONFIG_DYNAMIC_FTRACE
21#define ASSIGN_OPS_HASH(opsname, val) \
22 .func_hash = val, \
23 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
24#else
25#define ASSIGN_OPS_HASH(opsname, val)
26#endif
27
28DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph);
29int ftrace_graph_active;
30
31/* Both enabled by default (can be cleared by function_graph tracer flags */
32static bool fgraph_sleep_time = true;
33
34#ifdef CONFIG_DYNAMIC_FTRACE
35/*
36 * archs can override this function if they must do something
37 * to enable hook for graph tracer.
38 */
39int __weak ftrace_enable_ftrace_graph_caller(void)
40{
41 return 0;
42}
43
44/*
45 * archs can override this function if they must do something
46 * to disable hook for graph tracer.
47 */
48int __weak ftrace_disable_ftrace_graph_caller(void)
49{
50 return 0;
51}
52#endif
53
54/**
55 * ftrace_graph_stop - set to permanently disable function graph tracing
56 *
57 * In case of an error int function graph tracing, this is called
58 * to try to keep function graph tracing from causing any more harm.
59 * Usually this is pretty severe and this is called to try to at least
60 * get a warning out to the user.
61 */
62void ftrace_graph_stop(void)
63{
64 static_branch_enable(&kill_ftrace_graph);
65}
66
67/* Add a function return address to the trace stack on thread info.*/
68static int
69ftrace_push_return_trace(unsigned long ret, unsigned long func,
70 unsigned long frame_pointer, unsigned long *retp)
71{
72 unsigned long long calltime;
73 int index;
74
75 if (unlikely(ftrace_graph_is_dead()))
76 return -EBUSY;
77
78 if (!current->ret_stack)
79 return -EBUSY;
80
81 /*
82 * We must make sure the ret_stack is tested before we read
83 * anything else.
84 */
85 smp_rmb();
86
87 /* The return trace stack is full */
88 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
89 atomic_inc(¤t->trace_overrun);
90 return -EBUSY;
91 }
92
93 calltime = trace_clock_local();
94
95 index = ++current->curr_ret_stack;
96 barrier();
97 current->ret_stack[index].ret = ret;
98 current->ret_stack[index].func = func;
99 current->ret_stack[index].calltime = calltime;
100#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
101 current->ret_stack[index].fp = frame_pointer;
102#endif
103#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
104 current->ret_stack[index].retp = retp;
105#endif
106 return 0;
107}
108
109/*
110 * Not all archs define MCOUNT_INSN_SIZE which is used to look for direct
111 * functions. But those archs currently don't support direct functions
112 * anyway, and ftrace_find_rec_direct() is just a stub for them.
113 * Define MCOUNT_INSN_SIZE to keep those archs compiling.
114 */
115#ifndef MCOUNT_INSN_SIZE
116/* Make sure this only works without direct calls */
117# ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
118# error MCOUNT_INSN_SIZE not defined with direct calls enabled
119# endif
120# define MCOUNT_INSN_SIZE 0
121#endif
122
123int function_graph_enter(unsigned long ret, unsigned long func,
124 unsigned long frame_pointer, unsigned long *retp)
125{
126 struct ftrace_graph_ent trace;
127
128#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
129 /*
130 * Skip graph tracing if the return location is served by direct trampoline,
131 * since call sequence and return addresses are unpredictable anyway.
132 * Ex: BPF trampoline may call original function and may skip frame
133 * depending on type of BPF programs attached.
134 */
135 if (ftrace_direct_func_count &&
136 ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE))
137 return -EBUSY;
138#endif
139 trace.func = func;
140 trace.depth = ++current->curr_ret_depth;
141
142 if (ftrace_push_return_trace(ret, func, frame_pointer, retp))
143 goto out;
144
145 /* Only trace if the calling function expects to */
146 if (!ftrace_graph_entry(&trace))
147 goto out_ret;
148
149 return 0;
150 out_ret:
151 current->curr_ret_stack--;
152 out:
153 current->curr_ret_depth--;
154 return -EBUSY;
155}
156
157/* Retrieve a function return address to the trace stack on thread info.*/
158static void
159ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
160 unsigned long frame_pointer)
161{
162 int index;
163
164 index = current->curr_ret_stack;
165
166 if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) {
167 ftrace_graph_stop();
168 WARN_ON(1);
169 /* Might as well panic, otherwise we have no where to go */
170 *ret = (unsigned long)panic;
171 return;
172 }
173
174#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
175 /*
176 * The arch may choose to record the frame pointer used
177 * and check it here to make sure that it is what we expect it
178 * to be. If gcc does not set the place holder of the return
179 * address in the frame pointer, and does a copy instead, then
180 * the function graph trace will fail. This test detects this
181 * case.
182 *
183 * Currently, x86_32 with optimize for size (-Os) makes the latest
184 * gcc do the above.
185 *
186 * Note, -mfentry does not use frame pointers, and this test
187 * is not needed if CC_USING_FENTRY is set.
188 */
189 if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
190 ftrace_graph_stop();
191 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
192 " from func %ps return to %lx\n",
193 current->ret_stack[index].fp,
194 frame_pointer,
195 (void *)current->ret_stack[index].func,
196 current->ret_stack[index].ret);
197 *ret = (unsigned long)panic;
198 return;
199 }
200#endif
201
202 *ret = current->ret_stack[index].ret;
203 trace->func = current->ret_stack[index].func;
204 trace->calltime = current->ret_stack[index].calltime;
205 trace->overrun = atomic_read(¤t->trace_overrun);
206 trace->depth = current->curr_ret_depth--;
207 /*
208 * We still want to trace interrupts coming in if
209 * max_depth is set to 1. Make sure the decrement is
210 * seen before ftrace_graph_return.
211 */
212 barrier();
213}
214
215/*
216 * Hibernation protection.
217 * The state of the current task is too much unstable during
218 * suspend/restore to disk. We want to protect against that.
219 */
220static int
221ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
222 void *unused)
223{
224 switch (state) {
225 case PM_HIBERNATION_PREPARE:
226 pause_graph_tracing();
227 break;
228
229 case PM_POST_HIBERNATION:
230 unpause_graph_tracing();
231 break;
232 }
233 return NOTIFY_DONE;
234}
235
236static struct notifier_block ftrace_suspend_notifier = {
237 .notifier_call = ftrace_suspend_notifier_call,
238};
239
240/* fgraph_ret_regs is not defined without CONFIG_FUNCTION_GRAPH_RETVAL */
241struct fgraph_ret_regs;
242
243/*
244 * Send the trace to the ring-buffer.
245 * @return the original return address.
246 */
247static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs,
248 unsigned long frame_pointer)
249{
250 struct ftrace_graph_ret trace;
251 unsigned long ret;
252
253 ftrace_pop_return_trace(&trace, &ret, frame_pointer);
254#ifdef CONFIG_FUNCTION_GRAPH_RETVAL
255 trace.retval = fgraph_ret_regs_return_value(ret_regs);
256#endif
257 trace.rettime = trace_clock_local();
258 ftrace_graph_return(&trace);
259 /*
260 * The ftrace_graph_return() may still access the current
261 * ret_stack structure, we need to make sure the update of
262 * curr_ret_stack is after that.
263 */
264 barrier();
265 current->curr_ret_stack--;
266
267 if (unlikely(!ret)) {
268 ftrace_graph_stop();
269 WARN_ON(1);
270 /* Might as well panic. What else to do? */
271 ret = (unsigned long)panic;
272 }
273
274 return ret;
275}
276
277/*
278 * After all architecures have selected HAVE_FUNCTION_GRAPH_RETVAL, we can
279 * leave only ftrace_return_to_handler(ret_regs).
280 */
281#ifdef CONFIG_HAVE_FUNCTION_GRAPH_RETVAL
282unsigned long ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs)
283{
284 return __ftrace_return_to_handler(ret_regs,
285 fgraph_ret_regs_frame_pointer(ret_regs));
286}
287#else
288unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
289{
290 return __ftrace_return_to_handler(NULL, frame_pointer);
291}
292#endif
293
294/**
295 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
296 * @task: The task to read the shadow stack from
297 * @idx: Index down the shadow stack
298 *
299 * Return the ret_struct on the shadow stack of the @task at the
300 * call graph at @idx starting with zero. If @idx is zero, it
301 * will return the last saved ret_stack entry. If it is greater than
302 * zero, it will return the corresponding ret_stack for the depth
303 * of saved return addresses.
304 */
305struct ftrace_ret_stack *
306ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
307{
308 idx = task->curr_ret_stack - idx;
309
310 if (idx >= 0 && idx <= task->curr_ret_stack)
311 return &task->ret_stack[idx];
312
313 return NULL;
314}
315
316/**
317 * ftrace_graph_ret_addr - convert a potentially modified stack return address
318 * to its original value
319 *
320 * This function can be called by stack unwinding code to convert a found stack
321 * return address ('ret') to its original value, in case the function graph
322 * tracer has modified it to be 'return_to_handler'. If the address hasn't
323 * been modified, the unchanged value of 'ret' is returned.
324 *
325 * 'idx' is a state variable which should be initialized by the caller to zero
326 * before the first call.
327 *
328 * 'retp' is a pointer to the return address on the stack. It's ignored if
329 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
330 */
331#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
332unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
333 unsigned long ret, unsigned long *retp)
334{
335 int index = task->curr_ret_stack;
336 int i;
337
338 if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler))
339 return ret;
340
341 if (index < 0)
342 return ret;
343
344 for (i = 0; i <= index; i++)
345 if (task->ret_stack[i].retp == retp)
346 return task->ret_stack[i].ret;
347
348 return ret;
349}
350#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
351unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
352 unsigned long ret, unsigned long *retp)
353{
354 int task_idx;
355
356 if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler))
357 return ret;
358
359 task_idx = task->curr_ret_stack;
360
361 if (!task->ret_stack || task_idx < *idx)
362 return ret;
363
364 task_idx -= *idx;
365 (*idx)++;
366
367 return task->ret_stack[task_idx].ret;
368}
369#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
370
371static struct ftrace_ops graph_ops = {
372 .func = ftrace_graph_func,
373 .flags = FTRACE_OPS_FL_INITIALIZED |
374 FTRACE_OPS_FL_PID |
375 FTRACE_OPS_GRAPH_STUB,
376#ifdef FTRACE_GRAPH_TRAMP_ADDR
377 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
378 /* trampoline_size is only needed for dynamically allocated tramps */
379#endif
380 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
381};
382
383void ftrace_graph_sleep_time_control(bool enable)
384{
385 fgraph_sleep_time = enable;
386}
387
388int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
389{
390 return 0;
391}
392
393/*
394 * Simply points to ftrace_stub, but with the proper protocol.
395 * Defined by the linker script in linux/vmlinux.lds.h
396 */
397extern void ftrace_stub_graph(struct ftrace_graph_ret *);
398
399/* The callbacks that hook a function */
400trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph;
401trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
402static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
403
404/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
405static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
406{
407 int i;
408 int ret = 0;
409 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
410 struct task_struct *g, *t;
411
412 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
413 ret_stack_list[i] =
414 kmalloc_array(FTRACE_RETFUNC_DEPTH,
415 sizeof(struct ftrace_ret_stack),
416 GFP_KERNEL);
417 if (!ret_stack_list[i]) {
418 start = 0;
419 end = i;
420 ret = -ENOMEM;
421 goto free;
422 }
423 }
424
425 rcu_read_lock();
426 for_each_process_thread(g, t) {
427 if (start == end) {
428 ret = -EAGAIN;
429 goto unlock;
430 }
431
432 if (t->ret_stack == NULL) {
433 atomic_set(&t->trace_overrun, 0);
434 t->curr_ret_stack = -1;
435 t->curr_ret_depth = -1;
436 /* Make sure the tasks see the -1 first: */
437 smp_wmb();
438 t->ret_stack = ret_stack_list[start++];
439 }
440 }
441
442unlock:
443 rcu_read_unlock();
444free:
445 for (i = start; i < end; i++)
446 kfree(ret_stack_list[i]);
447 return ret;
448}
449
450static void
451ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
452 struct task_struct *prev,
453 struct task_struct *next,
454 unsigned int prev_state)
455{
456 unsigned long long timestamp;
457 int index;
458
459 /*
460 * Does the user want to count the time a function was asleep.
461 * If so, do not update the time stamps.
462 */
463 if (fgraph_sleep_time)
464 return;
465
466 timestamp = trace_clock_local();
467
468 prev->ftrace_timestamp = timestamp;
469
470 /* only process tasks that we timestamped */
471 if (!next->ftrace_timestamp)
472 return;
473
474 /*
475 * Update all the counters in next to make up for the
476 * time next was sleeping.
477 */
478 timestamp -= next->ftrace_timestamp;
479
480 for (index = next->curr_ret_stack; index >= 0; index--)
481 next->ret_stack[index].calltime += timestamp;
482}
483
484static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
485{
486 if (!ftrace_ops_test(&global_ops, trace->func, NULL))
487 return 0;
488 return __ftrace_graph_entry(trace);
489}
490
491/*
492 * The function graph tracer should only trace the functions defined
493 * by set_ftrace_filter and set_ftrace_notrace. If another function
494 * tracer ops is registered, the graph tracer requires testing the
495 * function against the global ops, and not just trace any function
496 * that any ftrace_ops registered.
497 */
498void update_function_graph_func(void)
499{
500 struct ftrace_ops *op;
501 bool do_test = false;
502
503 /*
504 * The graph and global ops share the same set of functions
505 * to test. If any other ops is on the list, then
506 * the graph tracing needs to test if its the function
507 * it should call.
508 */
509 do_for_each_ftrace_op(op, ftrace_ops_list) {
510 if (op != &global_ops && op != &graph_ops &&
511 op != &ftrace_list_end) {
512 do_test = true;
513 /* in double loop, break out with goto */
514 goto out;
515 }
516 } while_for_each_ftrace_op(op);
517 out:
518 if (do_test)
519 ftrace_graph_entry = ftrace_graph_entry_test;
520 else
521 ftrace_graph_entry = __ftrace_graph_entry;
522}
523
524static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
525
526static void
527graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
528{
529 atomic_set(&t->trace_overrun, 0);
530 t->ftrace_timestamp = 0;
531 /* make curr_ret_stack visible before we add the ret_stack */
532 smp_wmb();
533 t->ret_stack = ret_stack;
534}
535
536/*
537 * Allocate a return stack for the idle task. May be the first
538 * time through, or it may be done by CPU hotplug online.
539 */
540void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
541{
542 t->curr_ret_stack = -1;
543 t->curr_ret_depth = -1;
544 /*
545 * The idle task has no parent, it either has its own
546 * stack or no stack at all.
547 */
548 if (t->ret_stack)
549 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
550
551 if (ftrace_graph_active) {
552 struct ftrace_ret_stack *ret_stack;
553
554 ret_stack = per_cpu(idle_ret_stack, cpu);
555 if (!ret_stack) {
556 ret_stack =
557 kmalloc_array(FTRACE_RETFUNC_DEPTH,
558 sizeof(struct ftrace_ret_stack),
559 GFP_KERNEL);
560 if (!ret_stack)
561 return;
562 per_cpu(idle_ret_stack, cpu) = ret_stack;
563 }
564 graph_init_task(t, ret_stack);
565 }
566}
567
568/* Allocate a return stack for newly created task */
569void ftrace_graph_init_task(struct task_struct *t)
570{
571 /* Make sure we do not use the parent ret_stack */
572 t->ret_stack = NULL;
573 t->curr_ret_stack = -1;
574 t->curr_ret_depth = -1;
575
576 if (ftrace_graph_active) {
577 struct ftrace_ret_stack *ret_stack;
578
579 ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
580 sizeof(struct ftrace_ret_stack),
581 GFP_KERNEL);
582 if (!ret_stack)
583 return;
584 graph_init_task(t, ret_stack);
585 }
586}
587
588void ftrace_graph_exit_task(struct task_struct *t)
589{
590 struct ftrace_ret_stack *ret_stack = t->ret_stack;
591
592 t->ret_stack = NULL;
593 /* NULL must become visible to IRQs before we free it: */
594 barrier();
595
596 kfree(ret_stack);
597}
598
599/* Allocate a return stack for each task */
600static int start_graph_tracing(void)
601{
602 struct ftrace_ret_stack **ret_stack_list;
603 int ret, cpu;
604
605 ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
606 sizeof(struct ftrace_ret_stack *),
607 GFP_KERNEL);
608
609 if (!ret_stack_list)
610 return -ENOMEM;
611
612 /* The cpu_boot init_task->ret_stack will never be freed */
613 for_each_online_cpu(cpu) {
614 if (!idle_task(cpu)->ret_stack)
615 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
616 }
617
618 do {
619 ret = alloc_retstack_tasklist(ret_stack_list);
620 } while (ret == -EAGAIN);
621
622 if (!ret) {
623 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
624 if (ret)
625 pr_info("ftrace_graph: Couldn't activate tracepoint"
626 " probe to kernel_sched_switch\n");
627 }
628
629 kfree(ret_stack_list);
630 return ret;
631}
632
633int register_ftrace_graph(struct fgraph_ops *gops)
634{
635 int ret = 0;
636
637 mutex_lock(&ftrace_lock);
638
639 /* we currently allow only one tracer registered at a time */
640 if (ftrace_graph_active) {
641 ret = -EBUSY;
642 goto out;
643 }
644
645 register_pm_notifier(&ftrace_suspend_notifier);
646
647 ftrace_graph_active++;
648 ret = start_graph_tracing();
649 if (ret) {
650 ftrace_graph_active--;
651 goto out;
652 }
653
654 ftrace_graph_return = gops->retfunc;
655
656 /*
657 * Update the indirect function to the entryfunc, and the
658 * function that gets called to the entry_test first. Then
659 * call the update fgraph entry function to determine if
660 * the entryfunc should be called directly or not.
661 */
662 __ftrace_graph_entry = gops->entryfunc;
663 ftrace_graph_entry = ftrace_graph_entry_test;
664 update_function_graph_func();
665
666 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
667out:
668 mutex_unlock(&ftrace_lock);
669 return ret;
670}
671
672void unregister_ftrace_graph(struct fgraph_ops *gops)
673{
674 mutex_lock(&ftrace_lock);
675
676 if (unlikely(!ftrace_graph_active))
677 goto out;
678
679 ftrace_graph_active--;
680 ftrace_graph_return = ftrace_stub_graph;
681 ftrace_graph_entry = ftrace_graph_entry_stub;
682 __ftrace_graph_entry = ftrace_graph_entry_stub;
683 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
684 unregister_pm_notifier(&ftrace_suspend_notifier);
685 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
686
687 out:
688 mutex_unlock(&ftrace_lock);
689}