1/*
  2 *  linux/arch/x86_64/mcount_64.S
  3 *
  4 *  Copyright (C) 2014  Steven Rostedt, Red Hat Inc
  5 */
  6
  7#include <linux/linkage.h>
  8#include <asm/ptrace.h>
  9#include <asm/ftrace.h>
 10#include <asm/export.h>
 11
 12
 13	.code64
 14	.section .entry.text, "ax"
 15
 16
 17#ifdef CONFIG_FUNCTION_TRACER
 18
 19#ifdef CC_USING_FENTRY
 20# define function_hook	__fentry__
 21EXPORT_SYMBOL(__fentry__)
 22#else
 23# define function_hook	mcount
 24EXPORT_SYMBOL(mcount)
 25#endif
 26
 27/* All cases save the original rbp (8 bytes) */
 28#ifdef CONFIG_FRAME_POINTER
 29# ifdef CC_USING_FENTRY
 30/* Save parent and function stack frames (rip and rbp) */
 31#  define MCOUNT_FRAME_SIZE	(8+16*2)
 32# else
 33/* Save just function stack frame (rip and rbp) */
 34#  define MCOUNT_FRAME_SIZE	(8+16)
 35# endif
 36#else
 37/* No need to save a stack frame */
 38# define MCOUNT_FRAME_SIZE	8
 39#endif /* CONFIG_FRAME_POINTER */
 40
 41/* Size of stack used to save mcount regs in save_mcount_regs */
 42#define MCOUNT_REG_SIZE		(SS+8 + MCOUNT_FRAME_SIZE)
 43
 44/*
 45 * gcc -pg option adds a call to 'mcount' in most functions.
 46 * When -mfentry is used, the call is to 'fentry' and not 'mcount'
 47 * and is done before the function's stack frame is set up.
 48 * They both require a set of regs to be saved before calling
 49 * any C code and restored before returning back to the function.
 50 *
 51 * On boot up, all these calls are converted into nops. When tracing
 52 * is enabled, the call can jump to either ftrace_caller or
 53 * ftrace_regs_caller. Callbacks (tracing functions) that require
 54 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
 55 * it. For this reason, the size of the pt_regs structure will be
 56 * allocated on the stack and the required mcount registers will
 57 * be saved in the locations that pt_regs has them in.
 58 */
 59
 60/*
 61 * @added: the amount of stack added before calling this
 62 *
 63 * After this is called, the following registers contain:
 64 *
 65 *  %rdi - holds the address that called the trampoline
 66 *  %rsi - holds the parent function (traced function's return address)
 67 *  %rdx - holds the original %rbp
 68 */
 69.macro save_mcount_regs added=0
 70
 71	/* Always save the original rbp */
 72	pushq %rbp
 73
 74#ifdef CONFIG_FRAME_POINTER
 75	/*
 76	 * Stack traces will stop at the ftrace trampoline if the frame pointer
 77	 * is not set up properly. If fentry is used, we need to save a frame
 78	 * pointer for the parent as well as the function traced, because the
 79	 * fentry is called before the stack frame is set up, where as mcount
 80	 * is called afterward.
 81	 */
 82#ifdef CC_USING_FENTRY
 83	/* Save the parent pointer (skip orig rbp and our return address) */
 84	pushq \added+8*2(%rsp)
 85	pushq %rbp
 86	movq %rsp, %rbp
 87	/* Save the return address (now skip orig rbp, rbp and parent) */
 88	pushq \added+8*3(%rsp)
 89#else
 90	/* Can't assume that rip is before this (unless added was zero) */
 91	pushq \added+8(%rsp)
 92#endif
 93	pushq %rbp
 94	movq %rsp, %rbp
 95#endif /* CONFIG_FRAME_POINTER */
 96
 97	/*
 98	 * We add enough stack to save all regs.
 99	 */
100	subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
101	movq %rax, RAX(%rsp)
102	movq %rcx, RCX(%rsp)
103	movq %rdx, RDX(%rsp)
104	movq %rsi, RSI(%rsp)
105	movq %rdi, RDI(%rsp)
106	movq %r8, R8(%rsp)
107	movq %r9, R9(%rsp)
108	/*
109	 * Save the original RBP. Even though the mcount ABI does not
110	 * require this, it helps out callers.
111	 */
112	movq MCOUNT_REG_SIZE-8(%rsp), %rdx
113	movq %rdx, RBP(%rsp)
114
115	/* Copy the parent address into %rsi (second parameter) */
116#ifdef CC_USING_FENTRY
117	movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
118#else
119	/* %rdx contains original %rbp */
120	movq 8(%rdx), %rsi
121#endif
122
123	 /* Move RIP to its proper location */
124	movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
125	movq %rdi, RIP(%rsp)
126
127	/*
128	 * Now %rdi (the first parameter) has the return address of
129	 * where ftrace_call returns. But the callbacks expect the
130	 * address of the call itself.
131	 */
132	subq $MCOUNT_INSN_SIZE, %rdi
133	.endm
134
135.macro restore_mcount_regs
136	movq R9(%rsp), %r9
137	movq R8(%rsp), %r8
138	movq RDI(%rsp), %rdi
139	movq RSI(%rsp), %rsi
140	movq RDX(%rsp), %rdx
141	movq RCX(%rsp), %rcx
142	movq RAX(%rsp), %rax
143
144	/* ftrace_regs_caller can modify %rbp */
145	movq RBP(%rsp), %rbp
146
147	addq $MCOUNT_REG_SIZE, %rsp
148
149	.endm
150
151#ifdef CONFIG_DYNAMIC_FTRACE
152
153ENTRY(function_hook)
154	retq
155END(function_hook)
156
157ENTRY(ftrace_caller)
158	/* save_mcount_regs fills in first two parameters */
159	save_mcount_regs
160
161GLOBAL(ftrace_caller_op_ptr)
162	/* Load the ftrace_ops into the 3rd parameter */
163	movq function_trace_op(%rip), %rdx
164
165	/* regs go into 4th parameter (but make it NULL) */
166	movq $0, %rcx
167
168GLOBAL(ftrace_call)
169	call ftrace_stub
170
171	restore_mcount_regs
172
173	/*
174	 * The copied trampoline must call ftrace_epilogue as it
175	 * still may need to call the function graph tracer.
176	 *
177	 * The code up to this label is copied into trampolines so
178	 * think twice before adding any new code or changing the
179	 * layout here.
180	 */
181GLOBAL(ftrace_epilogue)
182
183#ifdef CONFIG_FUNCTION_GRAPH_TRACER
184GLOBAL(ftrace_graph_call)
185	jmp ftrace_stub
186#endif
187
188/* This is weak to keep gas from relaxing the jumps */
189WEAK(ftrace_stub)
190	retq
191END(ftrace_caller)
192
193ENTRY(ftrace_regs_caller)
194	/* Save the current flags before any operations that can change them */
195	pushfq
196
197	/* added 8 bytes to save flags */
198	save_mcount_regs 8
199	/* save_mcount_regs fills in first two parameters */
200
201GLOBAL(ftrace_regs_caller_op_ptr)
202	/* Load the ftrace_ops into the 3rd parameter */
203	movq function_trace_op(%rip), %rdx
204
205	/* Save the rest of pt_regs */
206	movq %r15, R15(%rsp)
207	movq %r14, R14(%rsp)
208	movq %r13, R13(%rsp)
209	movq %r12, R12(%rsp)
210	movq %r11, R11(%rsp)
211	movq %r10, R10(%rsp)
212	movq %rbx, RBX(%rsp)
213	/* Copy saved flags */
214	movq MCOUNT_REG_SIZE(%rsp), %rcx
215	movq %rcx, EFLAGS(%rsp)
216	/* Kernel segments */
217	movq $__KERNEL_DS, %rcx
218	movq %rcx, SS(%rsp)
219	movq $__KERNEL_CS, %rcx
220	movq %rcx, CS(%rsp)
221	/* Stack - skipping return address and flags */
222	leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
223	movq %rcx, RSP(%rsp)
224
225	/* regs go into 4th parameter */
226	leaq (%rsp), %rcx
227
228GLOBAL(ftrace_regs_call)
229	call ftrace_stub
230
231	/* Copy flags back to SS, to restore them */
232	movq EFLAGS(%rsp), %rax
233	movq %rax, MCOUNT_REG_SIZE(%rsp)
234
235	/* Handlers can change the RIP */
236	movq RIP(%rsp), %rax
237	movq %rax, MCOUNT_REG_SIZE+8(%rsp)
238
239	/* restore the rest of pt_regs */
240	movq R15(%rsp), %r15
241	movq R14(%rsp), %r14
242	movq R13(%rsp), %r13
243	movq R12(%rsp), %r12
244	movq R10(%rsp), %r10
245	movq RBX(%rsp), %rbx
246
247	restore_mcount_regs
248
249	/* Restore flags */
250	popfq
251
252	/*
253	 * As this jmp to ftrace_epilogue can be a short jump
254	 * it must not be copied into the trampoline.
255	 * The trampoline will add the code to jump
256	 * to the return.
257	 */
258GLOBAL(ftrace_regs_caller_end)
259
260	jmp ftrace_epilogue
261
262END(ftrace_regs_caller)
263
264
265#else /* ! CONFIG_DYNAMIC_FTRACE */
266
267ENTRY(function_hook)
268	cmpq $ftrace_stub, ftrace_trace_function
269	jnz trace
270
271fgraph_trace:
272#ifdef CONFIG_FUNCTION_GRAPH_TRACER
273	cmpq $ftrace_stub, ftrace_graph_return
274	jnz ftrace_graph_caller
275
276	cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
277	jnz ftrace_graph_caller
278#endif
279
280GLOBAL(ftrace_stub)
281	retq
282
283trace:
284	/* save_mcount_regs fills in first two parameters */
285	save_mcount_regs
286
287	/*
288	 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not
289	 * set (see include/asm/ftrace.h and include/linux/ftrace.h).  Only the
290	 * ip and parent ip are used and the list function is called when
291	 * function tracing is enabled.
292	 */
293	call   *ftrace_trace_function
294
295	restore_mcount_regs
296
297	jmp fgraph_trace
298END(function_hook)
299#endif /* CONFIG_DYNAMIC_FTRACE */
300#endif /* CONFIG_FUNCTION_TRACER */
301
302#ifdef CONFIG_FUNCTION_GRAPH_TRACER
303ENTRY(ftrace_graph_caller)
304	/* Saves rbp into %rdx and fills first parameter  */
305	save_mcount_regs
306
307#ifdef CC_USING_FENTRY
308	leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
309	movq $0, %rdx	/* No framepointers needed */
310#else
311	/* Save address of the return address of traced function */
312	leaq 8(%rdx), %rsi
313	/* ftrace does sanity checks against frame pointers */
314	movq (%rdx), %rdx
315#endif
316	call	prepare_ftrace_return
317
318	restore_mcount_regs
319
320	retq
321END(ftrace_graph_caller)
322
323GLOBAL(return_to_handler)
324	subq  $24, %rsp
325
326	/* Save the return values */
327	movq %rax, (%rsp)
328	movq %rdx, 8(%rsp)
329	movq %rbp, %rdi
330
331	call ftrace_return_to_handler
332
333	movq %rax, %rdi
334	movq 8(%rsp), %rdx
335	movq (%rsp), %rax
336	addq $24, %rsp
337	jmp *%rdi
338#endif