Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2017 Steven Rostedt, VMware Inc.
4 */
5
6#include <linux/linkage.h>
7#include <asm/page_types.h>
8#include <asm/segment.h>
9#include <asm/export.h>
10#include <asm/ftrace.h>
11#include <asm/nospec-branch.h>
12#include <asm/frame.h>
13#include <asm/asm-offsets.h>
14
15# define function_hook __fentry__
16EXPORT_SYMBOL(__fentry__)
17
18#ifdef CONFIG_FRAME_POINTER
19# define MCOUNT_FRAME 1 /* using frame = true */
20#else
21# define MCOUNT_FRAME 0 /* using frame = false */
22#endif
23
24ENTRY(function_hook)
25 ret
26END(function_hook)
27
28ENTRY(ftrace_caller)
29
30#ifdef CONFIG_FRAME_POINTER
31 /*
32 * Frame pointers are of ip followed by bp.
33 * Since fentry is an immediate jump, we are left with
34 * parent-ip, function-ip. We need to add a frame with
35 * parent-ip followed by ebp.
36 */
37 pushl 4(%esp) /* parent ip */
38 pushl %ebp
39 movl %esp, %ebp
40 pushl 2*4(%esp) /* function ip */
41
42 /* For mcount, the function ip is directly above */
43 pushl %ebp
44 movl %esp, %ebp
45#endif
46 pushl %eax
47 pushl %ecx
48 pushl %edx
49 pushl $0 /* Pass NULL as regs pointer */
50
51#ifdef CONFIG_FRAME_POINTER
52 /* Load parent ebp into edx */
53 movl 4*4(%esp), %edx
54#else
55 /* There's no frame pointer, load the appropriate stack addr instead */
56 lea 4*4(%esp), %edx
57#endif
58
59 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
60 /* Get the parent ip */
61 movl 4(%edx), %edx /* edx has ebp */
62
63 movl function_trace_op, %ecx
64 subl $MCOUNT_INSN_SIZE, %eax
65
66.globl ftrace_call
67ftrace_call:
68 call ftrace_stub
69
70 addl $4, %esp /* skip NULL pointer */
71 popl %edx
72 popl %ecx
73 popl %eax
74#ifdef CONFIG_FRAME_POINTER
75 popl %ebp
76 addl $4,%esp /* skip function ip */
77 popl %ebp /* this is the orig bp */
78 addl $4, %esp /* skip parent ip */
79#endif
80.Lftrace_ret:
81#ifdef CONFIG_FUNCTION_GRAPH_TRACER
82.globl ftrace_graph_call
83ftrace_graph_call:
84 jmp ftrace_stub
85#endif
86
87/* This is weak to keep gas from relaxing the jumps */
88WEAK(ftrace_stub)
89 ret
90END(ftrace_caller)
91
92ENTRY(ftrace_regs_caller)
93 /*
94 * We're here from an mcount/fentry CALL, and the stack frame looks like:
95 *
96 * <previous context>
97 * RET-IP
98 *
99 * The purpose of this function is to call out in an emulated INT3
100 * environment with a stack frame like:
101 *
102 * <previous context>
103 * gap / RET-IP
104 * gap
105 * gap
106 * gap
107 * pt_regs
108 *
109 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
110 */
111 subl $3*4, %esp # RET-IP + 3 gaps
112 pushl %ss # ss
113 pushl %esp # points at ss
114 addl $5*4, (%esp) # make it point at <previous context>
115 pushfl # flags
116 pushl $__KERNEL_CS # cs
117 pushl 7*4(%esp) # ip <- RET-IP
118 pushl $0 # orig_eax
119
120 pushl %gs
121 pushl %fs
122 pushl %es
123 pushl %ds
124
125 pushl %eax
126 pushl %ebp
127 pushl %edi
128 pushl %esi
129 pushl %edx
130 pushl %ecx
131 pushl %ebx
132
133 ENCODE_FRAME_POINTER
134
135 movl PT_EIP(%esp), %eax # 1st argument: IP
136 subl $MCOUNT_INSN_SIZE, %eax
137 movl 21*4(%esp), %edx # 2nd argument: parent ip
138 movl function_trace_op, %ecx # 3rd argument: ftrace_pos
139 pushl %esp # 4th argument: pt_regs
140
141GLOBAL(ftrace_regs_call)
142 call ftrace_stub
143
144 addl $4, %esp # skip 4th argument
145
146 /* place IP below the new SP */
147 movl PT_OLDESP(%esp), %eax
148 movl PT_EIP(%esp), %ecx
149 movl %ecx, -4(%eax)
150
151 /* place EAX below that */
152 movl PT_EAX(%esp), %ecx
153 movl %ecx, -8(%eax)
154
155 popl %ebx
156 popl %ecx
157 popl %edx
158 popl %esi
159 popl %edi
160 popl %ebp
161
162 lea -8(%eax), %esp
163 popl %eax
164
165 jmp .Lftrace_ret
166
167#ifdef CONFIG_FUNCTION_GRAPH_TRACER
168ENTRY(ftrace_graph_caller)
169 pushl %eax
170 pushl %ecx
171 pushl %edx
172 movl 3*4(%esp), %eax
173 /* Even with frame pointers, fentry doesn't have one here */
174 lea 4*4(%esp), %edx
175 movl $0, %ecx
176 subl $MCOUNT_INSN_SIZE, %eax
177 call prepare_ftrace_return
178 popl %edx
179 popl %ecx
180 popl %eax
181 ret
182END(ftrace_graph_caller)
183
184.globl return_to_handler
185return_to_handler:
186 pushl %eax
187 pushl %edx
188 movl $0, %eax
189 call ftrace_return_to_handler
190 movl %eax, %ecx
191 popl %edx
192 popl %eax
193 JMP_NOSPEC %ecx
194#endif
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2017 Steven Rostedt, VMware Inc.
4 */
5
6#include <linux/export.h>
7#include <linux/linkage.h>
8#include <asm/page_types.h>
9#include <asm/segment.h>
10#include <asm/ftrace.h>
11#include <asm/nospec-branch.h>
12#include <asm/frame.h>
13#include <asm/asm-offsets.h>
14
15#ifdef CONFIG_FRAME_POINTER
16# define MCOUNT_FRAME 1 /* using frame = true */
17#else
18# define MCOUNT_FRAME 0 /* using frame = false */
19#endif
20
21SYM_FUNC_START(__fentry__)
22 RET
23SYM_FUNC_END(__fentry__)
24EXPORT_SYMBOL(__fentry__)
25
26SYM_CODE_START(ftrace_caller)
27
28#ifdef CONFIG_FRAME_POINTER
29 /*
30 * Frame pointers are of ip followed by bp.
31 * Since fentry is an immediate jump, we are left with
32 * parent-ip, function-ip. We need to add a frame with
33 * parent-ip followed by ebp.
34 */
35 pushl 4(%esp) /* parent ip */
36 pushl %ebp
37 movl %esp, %ebp
38 pushl 2*4(%esp) /* function ip */
39
40 /* For mcount, the function ip is directly above */
41 pushl %ebp
42 movl %esp, %ebp
43#endif
44 pushl %eax
45 pushl %ecx
46 pushl %edx
47 pushl $0 /* Pass NULL as regs pointer */
48
49#ifdef CONFIG_FRAME_POINTER
50 /* Load parent ebp into edx */
51 movl 4*4(%esp), %edx
52#else
53 /* There's no frame pointer, load the appropriate stack addr instead */
54 lea 4*4(%esp), %edx
55#endif
56
57 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
58 /* Get the parent ip */
59 movl 4(%edx), %edx /* edx has ebp */
60
61 movl function_trace_op, %ecx
62 subl $MCOUNT_INSN_SIZE, %eax
63
64.globl ftrace_call
65ftrace_call:
66 call ftrace_stub
67
68 addl $4, %esp /* skip NULL pointer */
69 popl %edx
70 popl %ecx
71 popl %eax
72#ifdef CONFIG_FRAME_POINTER
73 popl %ebp
74 addl $4,%esp /* skip function ip */
75 popl %ebp /* this is the orig bp */
76 addl $4, %esp /* skip parent ip */
77#endif
78.Lftrace_ret:
79#ifdef CONFIG_FUNCTION_GRAPH_TRACER
80.globl ftrace_graph_call
81ftrace_graph_call:
82 jmp ftrace_stub
83#endif
84
85/* This is weak to keep gas from relaxing the jumps */
86SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
87 RET
88SYM_CODE_END(ftrace_caller)
89
90SYM_CODE_START(ftrace_regs_caller)
91 /*
92 * We're here from an mcount/fentry CALL, and the stack frame looks like:
93 *
94 * <previous context>
95 * RET-IP
96 *
97 * The purpose of this function is to call out in an emulated INT3
98 * environment with a stack frame like:
99 *
100 * <previous context>
101 * gap / RET-IP
102 * gap
103 * gap
104 * gap
105 * pt_regs
106 *
107 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
108 */
109 subl $3*4, %esp # RET-IP + 3 gaps
110 pushl %ss # ss
111 pushl %esp # points at ss
112 addl $5*4, (%esp) # make it point at <previous context>
113 pushfl # flags
114 pushl $__KERNEL_CS # cs
115 pushl 7*4(%esp) # ip <- RET-IP
116 pushl $0 # orig_eax
117
118 pushl %gs
119 pushl %fs
120 pushl %es
121 pushl %ds
122
123 pushl %eax
124 pushl %ebp
125 pushl %edi
126 pushl %esi
127 pushl %edx
128 pushl %ecx
129 pushl %ebx
130
131 ENCODE_FRAME_POINTER
132
133 movl PT_EIP(%esp), %eax # 1st argument: IP
134 subl $MCOUNT_INSN_SIZE, %eax
135 movl 21*4(%esp), %edx # 2nd argument: parent ip
136 movl function_trace_op, %ecx # 3rd argument: ftrace_pos
137 pushl %esp # 4th argument: pt_regs
138
139SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
140 call ftrace_stub
141
142 addl $4, %esp # skip 4th argument
143
144 /* place IP below the new SP */
145 movl PT_OLDESP(%esp), %eax
146 movl PT_EIP(%esp), %ecx
147 movl %ecx, -4(%eax)
148
149 /* place EAX below that */
150 movl PT_EAX(%esp), %ecx
151 movl %ecx, -8(%eax)
152
153 popl %ebx
154 popl %ecx
155 popl %edx
156 popl %esi
157 popl %edi
158 popl %ebp
159
160 lea -8(%eax), %esp
161 popl %eax
162
163 jmp .Lftrace_ret
164SYM_CODE_END(ftrace_regs_caller)
165
166SYM_FUNC_START(ftrace_stub_direct_tramp)
167 CALL_DEPTH_ACCOUNT
168 RET
169SYM_FUNC_END(ftrace_stub_direct_tramp)
170
171#ifdef CONFIG_FUNCTION_GRAPH_TRACER
172SYM_CODE_START(ftrace_graph_caller)
173 pushl %eax
174 pushl %ecx
175 pushl %edx
176 movl 3*4(%esp), %eax
177 /* Even with frame pointers, fentry doesn't have one here */
178 lea 4*4(%esp), %edx
179 movl $0, %ecx
180 subl $MCOUNT_INSN_SIZE, %eax
181 call prepare_ftrace_return
182 popl %edx
183 popl %ecx
184 popl %eax
185 RET
186SYM_CODE_END(ftrace_graph_caller)
187
188.globl return_to_handler
189return_to_handler:
190 pushl $0
191 pushl %edx
192 pushl %eax
193 movl %esp, %eax
194 call ftrace_return_to_handler
195 movl %eax, %ecx
196 popl %eax
197 popl %edx
198 addl $4, %esp # skip ebp
199 JMP_NOSPEC ecx
200#endif