1#include <asm/ptrace.h>
  2
  3#include "bpf_jit.h"
  4
  5#ifdef CONFIG_SPARC64
  6#define SAVE_SZ		176
  7#define SCRATCH_OFF	STACK_BIAS + 128
  8#define BE_PTR(label)	be,pn %xcc, label
  9#define SIGN_EXTEND(reg)	sra reg, 0, reg
 10#else
 11#define SAVE_SZ		96
 12#define SCRATCH_OFF	72
 13#define BE_PTR(label)	be label
 14#define SIGN_EXTEND(reg)
 15#endif
 16
 17#define SKF_MAX_NEG_OFF	(-0x200000) /* SKF_LL_OFF from filter.h */
 18
 19	.text
 20	.globl	bpf_jit_load_word
 21bpf_jit_load_word:
 22	cmp	r_OFF, 0
 23	bl	bpf_slow_path_word_neg
 24	 nop
 25	.globl	bpf_jit_load_word_positive_offset
 26bpf_jit_load_word_positive_offset:
 27	sub	r_HEADLEN, r_OFF, r_TMP
 28	cmp	r_TMP, 3
 29	ble	bpf_slow_path_word
 30	 add	r_SKB_DATA, r_OFF, r_TMP
 31	andcc	r_TMP, 3, %g0
 32	bne	load_word_unaligned
 33	 nop
 34	retl
 35	 ld	[r_TMP], r_A
 36load_word_unaligned:
 37	ldub	[r_TMP + 0x0], r_OFF
 38	ldub	[r_TMP + 0x1], r_TMP2
 39	sll	r_OFF, 8, r_OFF
 40	or	r_OFF, r_TMP2, r_OFF
 41	ldub	[r_TMP + 0x2], r_TMP2
 42	sll	r_OFF, 8, r_OFF
 43	or	r_OFF, r_TMP2, r_OFF
 44	ldub	[r_TMP + 0x3], r_TMP2
 45	sll	r_OFF, 8, r_OFF
 46	retl
 47	 or	r_OFF, r_TMP2, r_A
 48
 49	.globl	bpf_jit_load_half
 50bpf_jit_load_half:
 51	cmp	r_OFF, 0
 52	bl	bpf_slow_path_half_neg
 53	 nop
 54	.globl	bpf_jit_load_half_positive_offset
 55bpf_jit_load_half_positive_offset:
 56	sub	r_HEADLEN, r_OFF, r_TMP
 57	cmp	r_TMP, 1
 58	ble	bpf_slow_path_half
 59	 add	r_SKB_DATA, r_OFF, r_TMP
 60	andcc	r_TMP, 1, %g0
 61	bne	load_half_unaligned
 62	 nop
 63	retl
 64	 lduh	[r_TMP], r_A
 65load_half_unaligned:
 66	ldub	[r_TMP + 0x0], r_OFF
 67	ldub	[r_TMP + 0x1], r_TMP2
 68	sll	r_OFF, 8, r_OFF
 69	retl
 70	 or	r_OFF, r_TMP2, r_A
 71
 72	.globl	bpf_jit_load_byte
 73bpf_jit_load_byte:
 74	cmp	r_OFF, 0
 75	bl	bpf_slow_path_byte_neg
 76	 nop
 77	.globl	bpf_jit_load_byte_positive_offset
 78bpf_jit_load_byte_positive_offset:
 79	cmp	r_OFF, r_HEADLEN
 80	bge	bpf_slow_path_byte
 81	 nop
 82	retl
 83	 ldub	[r_SKB_DATA + r_OFF], r_A
 84
 85	.globl	bpf_jit_load_byte_msh
 86bpf_jit_load_byte_msh:
 87	cmp	r_OFF, 0
 88	bl	bpf_slow_path_byte_msh_neg
 89	 nop
 90	.globl	bpf_jit_load_byte_msh_positive_offset
 91bpf_jit_load_byte_msh_positive_offset:
 92	cmp	r_OFF, r_HEADLEN
 93	bge	bpf_slow_path_byte_msh
 94	 nop
 95	ldub	[r_SKB_DATA + r_OFF], r_OFF
 96	and	r_OFF, 0xf, r_OFF
 97	retl
 98	 sll	r_OFF, 2, r_X
 99
100#define bpf_slow_path_common(LEN)	\
101	save	%sp, -SAVE_SZ, %sp;	\
102	mov	%i0, %o0;		\
103	mov	r_OFF, %o1;		\
104	add	%fp, SCRATCH_OFF, %o2;	\
105	call	skb_copy_bits;		\
106	 mov	(LEN), %o3;		\
107	cmp	%o0, 0;			\
108	restore;
109
110bpf_slow_path_word:
111	bpf_slow_path_common(4)
112	bl	bpf_error
113	 ld	[%sp + SCRATCH_OFF], r_A
114	retl
115	 nop
116bpf_slow_path_half:
117	bpf_slow_path_common(2)
118	bl	bpf_error
119	 lduh	[%sp + SCRATCH_OFF], r_A
120	retl
121	 nop
122bpf_slow_path_byte:
123	bpf_slow_path_common(1)
124	bl	bpf_error
125	 ldub	[%sp + SCRATCH_OFF], r_A
126	retl
127	 nop
128bpf_slow_path_byte_msh:
129	bpf_slow_path_common(1)
130	bl	bpf_error
131	 ldub	[%sp + SCRATCH_OFF], r_A
132	and	r_OFF, 0xf, r_OFF
133	retl
134	 sll	r_OFF, 2, r_X
135
136#define bpf_negative_common(LEN)			\
137	save	%sp, -SAVE_SZ, %sp;			\
138	mov	%i0, %o0;				\
139	mov	r_OFF, %o1;				\
140	SIGN_EXTEND(%o1);				\
141	call	bpf_internal_load_pointer_neg_helper;	\
142	 mov	(LEN), %o2;				\
143	mov	%o0, r_TMP;				\
144	cmp	%o0, 0;					\
145	BE_PTR(bpf_error);				\
146	 restore;
147
148bpf_slow_path_word_neg:
149	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
150	cmp	r_OFF, r_TMP
151	bl	bpf_error
152	 nop
153	.globl	bpf_jit_load_word_negative_offset
154bpf_jit_load_word_negative_offset:
155	bpf_negative_common(4)
156	andcc	r_TMP, 3, %g0
157	bne	load_word_unaligned
158	 nop
159	retl
160	 ld	[r_TMP], r_A
161
162bpf_slow_path_half_neg:
163	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
164	cmp	r_OFF, r_TMP
165	bl	bpf_error
166	 nop
167	.globl	bpf_jit_load_half_negative_offset
168bpf_jit_load_half_negative_offset:
169	bpf_negative_common(2)
170	andcc	r_TMP, 1, %g0
171	bne	load_half_unaligned
172	 nop
173	retl
174	 lduh	[r_TMP], r_A
175
176bpf_slow_path_byte_neg:
177	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
178	cmp	r_OFF, r_TMP
179	bl	bpf_error
180	 nop
181	.globl	bpf_jit_load_byte_negative_offset
182bpf_jit_load_byte_negative_offset:
183	bpf_negative_common(1)
184	retl
185	 ldub	[r_TMP], r_A
186
187bpf_slow_path_byte_msh_neg:
188	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
189	cmp	r_OFF, r_TMP
190	bl	bpf_error
191	 nop
192	.globl	bpf_jit_load_byte_msh_negative_offset
193bpf_jit_load_byte_msh_negative_offset:
194	bpf_negative_common(1)
195	ldub	[r_TMP], r_OFF
196	and	r_OFF, 0xf, r_OFF
197	retl
198	 sll	r_OFF, 2, r_X
199
200bpf_error:
201	/* Make the JIT program return zero.  The JIT epilogue
202	 * stores away the original %o7 into r_saved_O7.  The
203	 * normal leaf function return is to use "retl" which
204	 * would evalute to "jmpl %o7 + 8, %g0" but we want to
205	 * use the saved value thus the sequence you see here.
206	 */
207	jmpl	r_saved_O7 + 8, %g0
208	 clr	%o0