1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <vmlinux.h>
  4#include <bpf/bpf_helpers.h>
  5#include "bpf_misc.h"
  6#include "bpf_experimental.h"
  7
  8/* From include/linux/filter.h */
  9#define MAX_BPF_STACK    512
 10
 11#if defined(__TARGET_ARCH_x86)
 12
 13struct elem {
 14	struct bpf_timer t;
 15	char pad[256];
 16};
 17
 18struct {
 19	__uint(type, BPF_MAP_TYPE_ARRAY);
 20	__uint(max_entries, 1);
 21	__type(key, int);
 22	__type(value, struct elem);
 23} array SEC(".maps");
 24
 25SEC("kprobe")
 26__description("Private stack, single prog")
 27__success
 28__arch_x86_64
 29__jited("	movabsq	$0x{{.*}}, %r9")
 30__jited("	addq	%gs:0x{{.*}}, %r9")
 31__jited("	movl	$0x2a, %edi")
 32__jited("	movq	%rdi, -0x100(%r9)")
 33__naked void private_stack_single_prog(void)
 34{
 35	asm volatile ("			\
 36	r1 = 42;			\
 37	*(u64 *)(r10 - 256) = r1;	\
 38	r0 = 0;				\
 39	exit;				\
 40"	::: __clobber_all);
 41}
 42
 43SEC("raw_tp")
 44__description("No private stack")
 45__success
 46__arch_x86_64
 47__jited("	subq	$0x8, %rsp")
 48__naked void no_private_stack_nested(void)
 49{
 50	asm volatile ("			\
 51	r1 = 42;			\
 52	*(u64 *)(r10 - 8) = r1;		\
 53	r0 = 0;				\
 54	exit;				\
 55"	::: __clobber_all);
 56}
 57
 58__used
 59__naked static void cumulative_stack_depth_subprog(void)
 60{
 61	asm volatile ("				\
 62	r1 = 41;				\
 63	*(u64 *)(r10 - 32) = r1;		\
 64	call %[bpf_get_smp_processor_id];	\
 65	exit;					\
 66"	:
 67	: __imm(bpf_get_smp_processor_id)
 68	: __clobber_all);
 69}
 70
 71SEC("kprobe")
 72__description("Private stack, subtree > MAX_BPF_STACK")
 73__success
 74__arch_x86_64
 75/* private stack fp for the main prog */
 76__jited("	movabsq	$0x{{.*}}, %r9")
 77__jited("	addq	%gs:0x{{.*}}, %r9")
 78__jited("	movl	$0x2a, %edi")
 79__jited("	movq	%rdi, -0x200(%r9)")
 80__jited("	pushq	%r9")
 81__jited("	callq	0x{{.*}}")
 82__jited("	popq	%r9")
 83__jited("	xorl	%eax, %eax")
 84__naked void private_stack_nested_1(void)
 85{
 86	asm volatile ("				\
 87	r1 = 42;				\
 88	*(u64 *)(r10 - %[max_bpf_stack]) = r1;	\
 89	call cumulative_stack_depth_subprog;	\
 90	r0 = 0;					\
 91	exit;					\
 92"	:
 93	: __imm_const(max_bpf_stack, MAX_BPF_STACK)
 94	: __clobber_all);
 95}
 96
 97__naked __noinline __used
 98static unsigned long loop_callback(void)
 99{
100	asm volatile ("				\
101	call %[bpf_get_prandom_u32];		\
102	r1 = 42;				\
103	*(u64 *)(r10 - 512) = r1;		\
104	call cumulative_stack_depth_subprog;	\
105	r0 = 0;					\
106	exit;					\
107"	:
108	: __imm(bpf_get_prandom_u32)
109	: __clobber_common);
110}
111
112SEC("raw_tp")
113__description("Private stack, callback")
114__success
115__arch_x86_64
116/* for func loop_callback */
117__jited("func #1")
118__jited("	endbr64")
119__jited("	nopl	(%rax,%rax)")
120__jited("	nopl	(%rax)")
121__jited("	pushq	%rbp")
122__jited("	movq	%rsp, %rbp")
123__jited("	endbr64")
124__jited("	movabsq	$0x{{.*}}, %r9")
125__jited("	addq	%gs:0x{{.*}}, %r9")
126__jited("	pushq	%r9")
127__jited("	callq")
128__jited("	popq	%r9")
129__jited("	movl	$0x2a, %edi")
130__jited("	movq	%rdi, -0x200(%r9)")
131__jited("	pushq	%r9")
132__jited("	callq")
133__jited("	popq	%r9")
134__naked void private_stack_callback(void)
135{
136	asm volatile ("			\
137	r1 = 1;				\
138	r2 = %[loop_callback];		\
139	r3 = 0;				\
140	r4 = 0;				\
141	call %[bpf_loop];		\
142	r0 = 0;				\
143	exit;				\
144"	:
145	: __imm_ptr(loop_callback),
146	  __imm(bpf_loop)
147	: __clobber_common);
148}
149
150SEC("fentry/bpf_fentry_test9")
151__description("Private stack, exception in main prog")
152__success __retval(0)
153__arch_x86_64
154__jited("	pushq	%r9")
155__jited("	callq")
156__jited("	popq	%r9")
157int private_stack_exception_main_prog(void)
158{
159	asm volatile ("			\
160	r1 = 42;			\
161	*(u64 *)(r10 - 512) = r1;	\
162"	::: __clobber_common);
163
164	bpf_throw(0);
165	return 0;
166}
167
168__used static int subprog_exception(void)
169{
170	bpf_throw(0);
171	return 0;
172}
173
174SEC("fentry/bpf_fentry_test9")
175__description("Private stack, exception in subprog")
176__success __retval(0)
177__arch_x86_64
178__jited("	movq	%rdi, -0x200(%r9)")
179__jited("	pushq	%r9")
180__jited("	callq")
181__jited("	popq	%r9")
182int private_stack_exception_sub_prog(void)
183{
184	asm volatile ("			\
185	r1 = 42;			\
186	*(u64 *)(r10 - 512) = r1;	\
187	call subprog_exception;		\
188"	::: __clobber_common);
189
190	return 0;
191}
192
193int glob;
194__noinline static void subprog2(int *val)
195{
196	glob += val[0] * 2;
197}
198
199__noinline static void subprog1(int *val)
200{
201	int tmp[64] = {};
202
203	tmp[0] = *val;
204	subprog2(tmp);
205}
206
207__noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
208{
209	subprog1(key);
210	return 0;
211}
212
213__noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
214{
215	return 0;
216}
217
218SEC("fentry/bpf_fentry_test9")
219__description("Private stack, async callback, not nested")
220__success __retval(0)
221__arch_x86_64
222__jited("	movabsq	$0x{{.*}}, %r9")
223int private_stack_async_callback_1(void)
224{
225	struct bpf_timer *arr_timer;
226	int array_key = 0;
227
228	arr_timer = bpf_map_lookup_elem(&array, &array_key);
229	if (!arr_timer)
230		return 0;
231
232	bpf_timer_init(arr_timer, &array, 1);
233	bpf_timer_set_callback(arr_timer, timer_cb2);
234	bpf_timer_start(arr_timer, 0, 0);
235	subprog1(&array_key);
236	return 0;
237}
238
239SEC("fentry/bpf_fentry_test9")
240__description("Private stack, async callback, potential nesting")
241__success __retval(0)
242__arch_x86_64
243__jited("	subq	$0x100, %rsp")
244int private_stack_async_callback_2(void)
245{
246	struct bpf_timer *arr_timer;
247	int array_key = 0;
248
249	arr_timer = bpf_map_lookup_elem(&array, &array_key);
250	if (!arr_timer)
251		return 0;
252
253	bpf_timer_init(arr_timer, &array, 1);
254	bpf_timer_set_callback(arr_timer, timer_cb1);
255	bpf_timer_start(arr_timer, 0, 0);
256	subprog1(&array_key);
257	return 0;
258}
259
260#else
261
262SEC("kprobe")
263__description("private stack is not supported, use a dummy test")
264__success
265int dummy_test(void)
266{
267	return 0;
268}
269
270#endif
271
272char _license[] SEC("license") = "GPL";