1/* SPDX-License-Identifier: GPL-2.0 */
  2#include <asm/ptrace.h>
  3
  4#include "bpf_jit_32.h"
  5
  6#define SAVE_SZ		96
  7#define SCRATCH_OFF	72
  8#define BE_PTR(label)	be label
  9#define SIGN_EXTEND(reg)
 10
 11#define SKF_MAX_NEG_OFF	(-0x200000) /* SKF_LL_OFF from filter.h */
 12
 13	.text
 14	.globl	bpf_jit_load_word
 15bpf_jit_load_word:
 16	cmp	r_OFF, 0
 17	bl	bpf_slow_path_word_neg
 18	 nop
 19	.globl	bpf_jit_load_word_positive_offset
 20bpf_jit_load_word_positive_offset:
 21	sub	r_HEADLEN, r_OFF, r_TMP
 22	cmp	r_TMP, 3
 23	ble	bpf_slow_path_word
 24	 add	r_SKB_DATA, r_OFF, r_TMP
 25	andcc	r_TMP, 3, %g0
 26	bne	load_word_unaligned
 27	 nop
 28	retl
 29	 ld	[r_TMP], r_A
 30load_word_unaligned:
 31	ldub	[r_TMP + 0x0], r_OFF
 32	ldub	[r_TMP + 0x1], r_TMP2
 33	sll	r_OFF, 8, r_OFF
 34	or	r_OFF, r_TMP2, r_OFF
 35	ldub	[r_TMP + 0x2], r_TMP2
 36	sll	r_OFF, 8, r_OFF
 37	or	r_OFF, r_TMP2, r_OFF
 38	ldub	[r_TMP + 0x3], r_TMP2
 39	sll	r_OFF, 8, r_OFF
 40	retl
 41	 or	r_OFF, r_TMP2, r_A
 42
 43	.globl	bpf_jit_load_half
 44bpf_jit_load_half:
 45	cmp	r_OFF, 0
 46	bl	bpf_slow_path_half_neg
 47	 nop
 48	.globl	bpf_jit_load_half_positive_offset
 49bpf_jit_load_half_positive_offset:
 50	sub	r_HEADLEN, r_OFF, r_TMP
 51	cmp	r_TMP, 1
 52	ble	bpf_slow_path_half
 53	 add	r_SKB_DATA, r_OFF, r_TMP
 54	andcc	r_TMP, 1, %g0
 55	bne	load_half_unaligned
 56	 nop
 57	retl
 58	 lduh	[r_TMP], r_A
 59load_half_unaligned:
 60	ldub	[r_TMP + 0x0], r_OFF
 61	ldub	[r_TMP + 0x1], r_TMP2
 62	sll	r_OFF, 8, r_OFF
 63	retl
 64	 or	r_OFF, r_TMP2, r_A
 65
 66	.globl	bpf_jit_load_byte
 67bpf_jit_load_byte:
 68	cmp	r_OFF, 0
 69	bl	bpf_slow_path_byte_neg
 70	 nop
 71	.globl	bpf_jit_load_byte_positive_offset
 72bpf_jit_load_byte_positive_offset:
 73	cmp	r_OFF, r_HEADLEN
 74	bge	bpf_slow_path_byte
 75	 nop
 76	retl
 77	 ldub	[r_SKB_DATA + r_OFF], r_A
 78
 79	.globl	bpf_jit_load_byte_msh
 80bpf_jit_load_byte_msh:
 81	cmp	r_OFF, 0
 82	bl	bpf_slow_path_byte_msh_neg
 83	 nop
 84	.globl	bpf_jit_load_byte_msh_positive_offset
 85bpf_jit_load_byte_msh_positive_offset:
 86	cmp	r_OFF, r_HEADLEN
 87	bge	bpf_slow_path_byte_msh
 88	 nop
 89	ldub	[r_SKB_DATA + r_OFF], r_OFF
 90	and	r_OFF, 0xf, r_OFF
 91	retl
 92	 sll	r_OFF, 2, r_X
 93
 94#define bpf_slow_path_common(LEN)	\
 95	save	%sp, -SAVE_SZ, %sp;	\
 96	mov	%i0, %o0;		\
 97	mov	r_OFF, %o1;		\
 98	add	%fp, SCRATCH_OFF, %o2;	\
 99	call	skb_copy_bits;		\
100	 mov	(LEN), %o3;		\
101	cmp	%o0, 0;			\
102	restore;
103
104bpf_slow_path_word:
105	bpf_slow_path_common(4)
106	bl	bpf_error
107	 ld	[%sp + SCRATCH_OFF], r_A
108	retl
109	 nop
110bpf_slow_path_half:
111	bpf_slow_path_common(2)
112	bl	bpf_error
113	 lduh	[%sp + SCRATCH_OFF], r_A
114	retl
115	 nop
116bpf_slow_path_byte:
117	bpf_slow_path_common(1)
118	bl	bpf_error
119	 ldub	[%sp + SCRATCH_OFF], r_A
120	retl
121	 nop
122bpf_slow_path_byte_msh:
123	bpf_slow_path_common(1)
124	bl	bpf_error
125	 ldub	[%sp + SCRATCH_OFF], r_A
126	and	r_OFF, 0xf, r_OFF
127	retl
128	 sll	r_OFF, 2, r_X
129
130#define bpf_negative_common(LEN)			\
131	save	%sp, -SAVE_SZ, %sp;			\
132	mov	%i0, %o0;				\
133	mov	r_OFF, %o1;				\
134	SIGN_EXTEND(%o1);				\
135	call	bpf_internal_load_pointer_neg_helper;	\
136	 mov	(LEN), %o2;				\
137	mov	%o0, r_TMP;				\
138	cmp	%o0, 0;					\
139	BE_PTR(bpf_error);				\
140	 restore;
141
142bpf_slow_path_word_neg:
143	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
144	cmp	r_OFF, r_TMP
145	bl	bpf_error
146	 nop
147	.globl	bpf_jit_load_word_negative_offset
148bpf_jit_load_word_negative_offset:
149	bpf_negative_common(4)
150	andcc	r_TMP, 3, %g0
151	bne	load_word_unaligned
152	 nop
153	retl
154	 ld	[r_TMP], r_A
155
156bpf_slow_path_half_neg:
157	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
158	cmp	r_OFF, r_TMP
159	bl	bpf_error
160	 nop
161	.globl	bpf_jit_load_half_negative_offset
162bpf_jit_load_half_negative_offset:
163	bpf_negative_common(2)
164	andcc	r_TMP, 1, %g0
165	bne	load_half_unaligned
166	 nop
167	retl
168	 lduh	[r_TMP], r_A
169
170bpf_slow_path_byte_neg:
171	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
172	cmp	r_OFF, r_TMP
173	bl	bpf_error
174	 nop
175	.globl	bpf_jit_load_byte_negative_offset
176bpf_jit_load_byte_negative_offset:
177	bpf_negative_common(1)
178	retl
179	 ldub	[r_TMP], r_A
180
181bpf_slow_path_byte_msh_neg:
182	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
183	cmp	r_OFF, r_TMP
184	bl	bpf_error
185	 nop
186	.globl	bpf_jit_load_byte_msh_negative_offset
187bpf_jit_load_byte_msh_negative_offset:
188	bpf_negative_common(1)
189	ldub	[r_TMP], r_OFF
190	and	r_OFF, 0xf, r_OFF
191	retl
192	 sll	r_OFF, 2, r_X
193
194bpf_error:
195	/* Make the JIT program return zero.  The JIT epilogue
196	 * stores away the original %o7 into r_saved_O7.  The
197	 * normal leaf function return is to use "retl" which
198	 * would evalute to "jmpl %o7 + 8, %g0" but we want to
199	 * use the saved value thus the sequence you see here.
200	 */
201	jmpl	r_saved_O7 + 8, %g0
202	 clr	%o0