1/* SPDX-License-Identifier: GPL-2.0-only */
  2
  3#include <asm/assembler.h>
  4#include <asm/ftrace.h>
  5#include <asm/unwind.h>
  6
  7#include "entry-header.S"
  8
  9/*
 10 * When compiling with -pg, gcc inserts a call to the mcount routine at the
 11 * start of every function.  In mcount, apart from the function's address (in
 12 * lr), we need to get hold of the function's caller's address.
 13 *
 14 * Newer GCCs (4.4+) solve this problem by using a version of mcount with call
 15 * sites like:
 16 *
 17 *	push	{lr}
 18 *	bl	__gnu_mcount_nc
 19 *
 20 * With these compilers, frame pointers are not necessary.
 21 *
 22 * mcount can be thought of as a function called in the middle of a subroutine
 23 * call.  As such, it needs to be transparent for both the caller and the
 24 * callee: the original lr needs to be restored when leaving mcount, and no
 25 * registers should be clobbered.  (In the __gnu_mcount_nc implementation, we
 26 * clobber the ip register.  This is OK because the ARM calling convention
 27 * allows it to be clobbered in subroutines and doesn't use it to hold
 28 * parameters.)
 29 *
 30 * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}"
 31 * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
 32 */
 33
 34.macro mcount_adjust_addr rd, rn
 35	bic	\rd, \rn, #1		@ clear the Thumb bit if present
 36	sub	\rd, \rd, #MCOUNT_INSN_SIZE
 37.endm
 38
 39.macro __mcount suffix
 40	mcount_enter
 41	ldr	r0, =ftrace_trace_function
 42	ldr	r2, [r0]
 43	adr	r0, .Lftrace_stub
 44	cmp	r0, r2
 45	bne	1f
 46
 47#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 48	ldr     r1, =ftrace_graph_return
 49	ldr     r2, [r1]
 50	cmp     r0, r2
 51	bne     ftrace_graph_caller\suffix
 52
 53	ldr     r1, =ftrace_graph_entry
 54	ldr     r2, [r1]
 55	ldr     r0, =ftrace_graph_entry_stub
 56	cmp     r0, r2
 57	bne     ftrace_graph_caller\suffix
 58#endif
 59
 60	mcount_exit
 61
 621: 	mcount_get_lr	r1			@ lr of instrumented func
 63	mcount_adjust_addr	r0, lr		@ instrumented function
 64	badr	lr, 2f
 65	mov	pc, r2
 662:	mcount_exit
 67.endm
 68
 69#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 70
 71.macro __ftrace_regs_caller
 72
 73	sub	sp, sp, #8	@ space for PC and CPSR OLD_R0,
 74				@ OLD_R0 will overwrite previous LR
 75
 76	add 	ip, sp, #12	@ move in IP the value of SP as it was
 77				@ before the push {lr} of the mcount mechanism
 78
 79	str     lr, [sp, #0]    @ store LR instead of PC
 80
 81	ldr     lr, [sp, #8]    @ get previous LR
 82
 83	str	r0, [sp, #8]	@ write r0 as OLD_R0 over previous LR
 84
 85	stmdb   sp!, {ip, lr}
 86	stmdb   sp!, {r0-r11, lr}
 87
 88	@ stack content at this point:
 89	@ 0  4          48   52       56            60   64    68       72
 90	@ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 |
 91
 92	mov r3, sp				@ struct pt_regs*
 93
 94	ldr r2, =function_trace_op
 95	ldr r2, [r2]				@ pointer to the current
 96						@ function tracing op
 97
 98	ldr	r1, [sp, #S_LR]			@ lr of instrumented func
 99
100	ldr	lr, [sp, #S_PC]			@ get LR
101
102	mcount_adjust_addr	r0, lr		@ instrumented function
103
104	.globl ftrace_regs_call
105ftrace_regs_call:
106	bl	ftrace_stub
107
108#ifdef CONFIG_FUNCTION_GRAPH_TRACER
109	.globl ftrace_graph_regs_call
110ftrace_graph_regs_call:
111	mov	r0, r0
112#endif
113
114	@ pop saved regs
115	ldmia   sp!, {r0-r12}			@ restore r0 through r12
116	ldr	ip, [sp, #8]			@ restore PC
117	ldr	lr, [sp, #4]			@ restore LR
118	ldr	sp, [sp, #0]			@ restore SP
119	mov	pc, ip				@ return
120.endm
121
122#ifdef CONFIG_FUNCTION_GRAPH_TRACER
123.macro __ftrace_graph_regs_caller
124
125	sub     r0, fp, #4              @ lr of instrumented routine (parent)
126
127	@ called from __ftrace_regs_caller
128	ldr     r1, [sp, #S_PC]		@ instrumented routine (func)
129	mcount_adjust_addr	r1, r1
130
131	mov	r2, fp			@ frame pointer
132	bl	prepare_ftrace_return
133
134	@ pop registers saved in ftrace_regs_caller
135	ldmia   sp!, {r0-r12}			@ restore r0 through r12
136	ldr	ip, [sp, #8]			@ restore PC
137	ldr	lr, [sp, #4]			@ restore LR
138	ldr	sp, [sp, #0]			@ restore SP
139	mov	pc, ip				@ return
140
141.endm
142#endif
143#endif
144
145.macro __ftrace_caller suffix
146	mcount_enter
147
148	mcount_get_lr	r1			@ lr of instrumented func
149	mcount_adjust_addr	r0, lr		@ instrumented function
150
151#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
152	ldr r2, =function_trace_op
153	ldr r2, [r2]				@ pointer to the current
154						@ function tracing op
155	mov r3, #0				@ regs is NULL
156#endif
157
158	.globl ftrace_call\suffix
159ftrace_call\suffix:
160	bl	ftrace_stub
161
162#ifdef CONFIG_FUNCTION_GRAPH_TRACER
163	.globl ftrace_graph_call\suffix
164ftrace_graph_call\suffix:
165	mov	r0, r0
166#endif
167
168	mcount_exit
169.endm
170
171.macro __ftrace_graph_caller
172	sub	r0, fp, #4		@ &lr of instrumented routine (&parent)
173#ifdef CONFIG_DYNAMIC_FTRACE
174	@ called from __ftrace_caller, saved in mcount_enter
175	ldr	r1, [sp, #16]		@ instrumented routine (func)
176	mcount_adjust_addr	r1, r1
177#else
178	@ called from __mcount, untouched in lr
179	mcount_adjust_addr	r1, lr	@ instrumented routine (func)
180#endif
181	mov	r2, fp			@ frame pointer
182	bl	prepare_ftrace_return
183	mcount_exit
184.endm
185
186/*
187 * __gnu_mcount_nc
188 */
189
190.macro mcount_enter
191/*
192 * This pad compensates for the push {lr} at the call site.  Note that we are
193 * unable to unwind through a function which does not otherwise save its lr.
194 */
195 UNWIND(.pad	#4)
196	stmdb	sp!, {r0-r3, lr}
197 UNWIND(.save	{r0-r3, lr})
198.endm
199
200.macro mcount_get_lr reg
201	ldr	\reg, [sp, #20]
202.endm
203
204.macro mcount_exit
205	ldmia	sp!, {r0-r3, ip, lr}
206	ret	ip
207.endm
208
209ENTRY(__gnu_mcount_nc)
210UNWIND(.fnstart)
211#ifdef CONFIG_DYNAMIC_FTRACE
212	mov	ip, lr
213	ldmia	sp!, {lr}
214	ret	ip
215#else
216	__mcount
217#endif
218UNWIND(.fnend)
219ENDPROC(__gnu_mcount_nc)
220
221#ifdef CONFIG_DYNAMIC_FTRACE
222ENTRY(ftrace_caller)
223UNWIND(.fnstart)
224	__ftrace_caller
225UNWIND(.fnend)
226ENDPROC(ftrace_caller)
227
228#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
229ENTRY(ftrace_regs_caller)
230UNWIND(.fnstart)
231	__ftrace_regs_caller
232UNWIND(.fnend)
233ENDPROC(ftrace_regs_caller)
234#endif
235
236#endif
237
238#ifdef CONFIG_FUNCTION_GRAPH_TRACER
239ENTRY(ftrace_graph_caller)
240UNWIND(.fnstart)
241	__ftrace_graph_caller
242UNWIND(.fnend)
243ENDPROC(ftrace_graph_caller)
244
245#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
246ENTRY(ftrace_graph_regs_caller)
247UNWIND(.fnstart)
248	__ftrace_graph_regs_caller
249UNWIND(.fnend)
250ENDPROC(ftrace_graph_regs_caller)
251#endif
252#endif
253
254.purgem mcount_enter
255.purgem mcount_get_lr
256.purgem mcount_exit
257
258#ifdef CONFIG_FUNCTION_GRAPH_TRACER
259	.globl return_to_handler
260return_to_handler:
261	stmdb	sp!, {r0-r3}
262	mov	r0, fp			@ frame pointer
263	bl	ftrace_return_to_handler
264	mov	lr, r0			@ r0 has real ret addr
265	ldmia	sp!, {r0-r3}
266	ret	lr
267#endif
268
269ENTRY(ftrace_stub)
270.Lftrace_stub:
271	ret	lr
272ENDPROC(ftrace_stub)