1/*
  2 *  linux/arch/x86_64/mcount_64.S
  3 *
  4 *  Copyright (C) 2014  Steven Rostedt, Red Hat Inc
  5 */
  6
  7#include <linux/linkage.h>
  8#include <asm/ptrace.h>
  9#include <asm/ftrace.h>
 10
 11
 12	.code64
 13	.section .entry.text, "ax"
 14
 15
 16#ifdef CONFIG_FUNCTION_TRACER
 17
 18#ifdef CC_USING_FENTRY
 19# define function_hook	__fentry__
 20#else
 21# define function_hook	mcount
 22#endif
 23
 24/* All cases save the original rbp (8 bytes) */
 25#ifdef CONFIG_FRAME_POINTER
 26# ifdef CC_USING_FENTRY
 27/* Save parent and function stack frames (rip and rbp) */
 28#  define MCOUNT_FRAME_SIZE	(8+16*2)
 29# else
 30/* Save just function stack frame (rip and rbp) */
 31#  define MCOUNT_FRAME_SIZE	(8+16)
 32# endif
 33#else
 34/* No need to save a stack frame */
 35# define MCOUNT_FRAME_SIZE	8
 36#endif /* CONFIG_FRAME_POINTER */
 37
 38/* Size of stack used to save mcount regs in save_mcount_regs */
 39#define MCOUNT_REG_SIZE		(SS+8 + MCOUNT_FRAME_SIZE)
 40
 41/*
 42 * gcc -pg option adds a call to 'mcount' in most functions.
 43 * When -mfentry is used, the call is to 'fentry' and not 'mcount'
 44 * and is done before the function's stack frame is set up.
 45 * They both require a set of regs to be saved before calling
 46 * any C code and restored before returning back to the function.
 47 *
 48 * On boot up, all these calls are converted into nops. When tracing
 49 * is enabled, the call can jump to either ftrace_caller or
 50 * ftrace_regs_caller. Callbacks (tracing functions) that require
 51 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
 52 * it. For this reason, the size of the pt_regs structure will be
 53 * allocated on the stack and the required mcount registers will
 54 * be saved in the locations that pt_regs has them in.
 55 */
 56
 57/*
 58 * @added: the amount of stack added before calling this
 59 *
 60 * After this is called, the following registers contain:
 61 *
 62 *  %rdi - holds the address that called the trampoline
 63 *  %rsi - holds the parent function (traced function's return address)
 64 *  %rdx - holds the original %rbp
 65 */
 66.macro save_mcount_regs added=0
 67
 68	/* Always save the original rbp */
 69	pushq %rbp
 70
 71#ifdef CONFIG_FRAME_POINTER
 72	/*
 73	 * Stack traces will stop at the ftrace trampoline if the frame pointer
 74	 * is not set up properly. If fentry is used, we need to save a frame
 75	 * pointer for the parent as well as the function traced, because the
 76	 * fentry is called before the stack frame is set up, where as mcount
 77	 * is called afterward.
 78	 */
 79#ifdef CC_USING_FENTRY
 80	/* Save the parent pointer (skip orig rbp and our return address) */
 81	pushq \added+8*2(%rsp)
 82	pushq %rbp
 83	movq %rsp, %rbp
 84	/* Save the return address (now skip orig rbp, rbp and parent) */
 85	pushq \added+8*3(%rsp)
 86#else
 87	/* Can't assume that rip is before this (unless added was zero) */
 88	pushq \added+8(%rsp)
 89#endif
 90	pushq %rbp
 91	movq %rsp, %rbp
 92#endif /* CONFIG_FRAME_POINTER */
 93
 94	/*
 95	 * We add enough stack to save all regs.
 96	 */
 97	subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
 98	movq %rax, RAX(%rsp)
 99	movq %rcx, RCX(%rsp)
100	movq %rdx, RDX(%rsp)
101	movq %rsi, RSI(%rsp)
102	movq %rdi, RDI(%rsp)
103	movq %r8, R8(%rsp)
104	movq %r9, R9(%rsp)
105	/*
106	 * Save the original RBP. Even though the mcount ABI does not
107	 * require this, it helps out callers.
108	 */
109	movq MCOUNT_REG_SIZE-8(%rsp), %rdx
110	movq %rdx, RBP(%rsp)
111
112	/* Copy the parent address into %rsi (second parameter) */
113#ifdef CC_USING_FENTRY
114	movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
115#else
116	/* %rdx contains original %rbp */
117	movq 8(%rdx), %rsi
118#endif
119
120	 /* Move RIP to its proper location */
121	movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
122	movq %rdi, RIP(%rsp)
123
124	/*
125	 * Now %rdi (the first parameter) has the return address of
126	 * where ftrace_call returns. But the callbacks expect the
127	 * address of the call itself.
128	 */
129	subq $MCOUNT_INSN_SIZE, %rdi
130	.endm
131
132.macro restore_mcount_regs
133	movq R9(%rsp), %r9
134	movq R8(%rsp), %r8
135	movq RDI(%rsp), %rdi
136	movq RSI(%rsp), %rsi
137	movq RDX(%rsp), %rdx
138	movq RCX(%rsp), %rcx
139	movq RAX(%rsp), %rax
140
141	/* ftrace_regs_caller can modify %rbp */
142	movq RBP(%rsp), %rbp
143
144	addq $MCOUNT_REG_SIZE, %rsp
145
146	.endm
147
148#ifdef CONFIG_DYNAMIC_FTRACE
149
150ENTRY(function_hook)
151	retq
152END(function_hook)
153
154ENTRY(ftrace_caller)
155	/* save_mcount_regs fills in first two parameters */
156	save_mcount_regs
157
158GLOBAL(ftrace_caller_op_ptr)
159	/* Load the ftrace_ops into the 3rd parameter */
160	movq function_trace_op(%rip), %rdx
161
162	/* regs go into 4th parameter (but make it NULL) */
163	movq $0, %rcx
164
165GLOBAL(ftrace_call)
166	call ftrace_stub
167
168	restore_mcount_regs
169
170	/*
171	 * The copied trampoline must call ftrace_epilogue as it
172	 * still may need to call the function graph tracer.
173	 *
174	 * The code up to this label is copied into trampolines so
175	 * think twice before adding any new code or changing the
176	 * layout here.
177	 */
178GLOBAL(ftrace_epilogue)
179
180#ifdef CONFIG_FUNCTION_GRAPH_TRACER
181GLOBAL(ftrace_graph_call)
182	jmp ftrace_stub
183#endif
184
185GLOBAL(ftrace_stub)
186	retq
187END(ftrace_caller)
188
189ENTRY(ftrace_regs_caller)
190	/* Save the current flags before any operations that can change them */
191	pushfq
192
193	/* added 8 bytes to save flags */
194	save_mcount_regs 8
195	/* save_mcount_regs fills in first two parameters */
196
197GLOBAL(ftrace_regs_caller_op_ptr)
198	/* Load the ftrace_ops into the 3rd parameter */
199	movq function_trace_op(%rip), %rdx
200
201	/* Save the rest of pt_regs */
202	movq %r15, R15(%rsp)
203	movq %r14, R14(%rsp)
204	movq %r13, R13(%rsp)
205	movq %r12, R12(%rsp)
206	movq %r11, R11(%rsp)
207	movq %r10, R10(%rsp)
208	movq %rbx, RBX(%rsp)
209	/* Copy saved flags */
210	movq MCOUNT_REG_SIZE(%rsp), %rcx
211	movq %rcx, EFLAGS(%rsp)
212	/* Kernel segments */
213	movq $__KERNEL_DS, %rcx
214	movq %rcx, SS(%rsp)
215	movq $__KERNEL_CS, %rcx
216	movq %rcx, CS(%rsp)
217	/* Stack - skipping return address and flags */
218	leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
219	movq %rcx, RSP(%rsp)
220
221	/* regs go into 4th parameter */
222	leaq (%rsp), %rcx
223
224GLOBAL(ftrace_regs_call)
225	call ftrace_stub
226
227	/* Copy flags back to SS, to restore them */
228	movq EFLAGS(%rsp), %rax
229	movq %rax, MCOUNT_REG_SIZE(%rsp)
230
231	/* Handlers can change the RIP */
232	movq RIP(%rsp), %rax
233	movq %rax, MCOUNT_REG_SIZE+8(%rsp)
234
235	/* restore the rest of pt_regs */
236	movq R15(%rsp), %r15
237	movq R14(%rsp), %r14
238	movq R13(%rsp), %r13
239	movq R12(%rsp), %r12
240	movq R10(%rsp), %r10
241	movq RBX(%rsp), %rbx
242
243	restore_mcount_regs
244
245	/* Restore flags */
246	popfq
247
248	/*
249	 * As this jmp to ftrace_epilogue can be a short jump
250	 * it must not be copied into the trampoline.
251	 * The trampoline will add the code to jump
252	 * to the return.
253	 */
254GLOBAL(ftrace_regs_caller_end)
255
256	jmp ftrace_epilogue
257
258END(ftrace_regs_caller)
259
260
261#else /* ! CONFIG_DYNAMIC_FTRACE */
262
263ENTRY(function_hook)
264	cmpq $ftrace_stub, ftrace_trace_function
265	jnz trace
266
267fgraph_trace:
268#ifdef CONFIG_FUNCTION_GRAPH_TRACER
269	cmpq $ftrace_stub, ftrace_graph_return
270	jnz ftrace_graph_caller
271
272	cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
273	jnz ftrace_graph_caller
274#endif
275
276GLOBAL(ftrace_stub)
277	retq
278
279trace:
280	/* save_mcount_regs fills in first two parameters */
281	save_mcount_regs
282
283	/*
284	 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not
285	 * set (see include/asm/ftrace.h and include/linux/ftrace.h).  Only the
286	 * ip and parent ip are used and the list function is called when
287	 * function tracing is enabled.
288	 */
289	call   *ftrace_trace_function
290
291	restore_mcount_regs
292
293	jmp fgraph_trace
294END(function_hook)
295#endif /* CONFIG_DYNAMIC_FTRACE */
296#endif /* CONFIG_FUNCTION_TRACER */
297
298#ifdef CONFIG_FUNCTION_GRAPH_TRACER
299ENTRY(ftrace_graph_caller)
300	/* Saves rbp into %rdx and fills first parameter  */
301	save_mcount_regs
302
303#ifdef CC_USING_FENTRY
304	leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
305	movq $0, %rdx	/* No framepointers needed */
306#else
307	/* Save address of the return address of traced function */
308	leaq 8(%rdx), %rsi
309	/* ftrace does sanity checks against frame pointers */
310	movq (%rdx), %rdx
311#endif
312	call	prepare_ftrace_return
313
314	restore_mcount_regs
315
316	retq
317END(ftrace_graph_caller)
318
319GLOBAL(return_to_handler)
320	subq  $24, %rsp
321
322	/* Save the return values */
323	movq %rax, (%rsp)
324	movq %rdx, 8(%rsp)
325	movq %rbp, %rdi
326
327	call ftrace_return_to_handler
328
329	movq %rax, %rdi
330	movq 8(%rsp), %rdx
331	movq (%rsp), %rax
332	addq $24, %rsp
333	jmp *%rdi
334#endif