1/* SPDX-License-Identifier: GPL-2.0 */
  2#include <asm/ptrace.h>
  3
  4#include "bpf_jit_64.h"
  5
  6#define SAVE_SZ		176
  7#define SCRATCH_OFF	STACK_BIAS + 128
  8#define BE_PTR(label)	be,pn %xcc, label
  9#define SIGN_EXTEND(reg)	sra reg, 0, reg
 10
 11#define SKF_MAX_NEG_OFF	(-0x200000) /* SKF_LL_OFF from filter.h */
 12
 13	.text
 14	.globl	bpf_jit_load_word
 15bpf_jit_load_word:
 16	cmp	r_OFF, 0
 17	bl	bpf_slow_path_word_neg
 18	 nop
 19	.globl	bpf_jit_load_word_positive_offset
 20bpf_jit_load_word_positive_offset:
 21	sub	r_HEADLEN, r_OFF, r_TMP
 22	cmp	r_TMP, 3
 23	ble	bpf_slow_path_word
 24	 add	r_SKB_DATA, r_OFF, r_TMP
 25	andcc	r_TMP, 3, %g0
 26	bne	load_word_unaligned
 27	 nop
 28	retl
 29	 ld	[r_TMP], r_RESULT
 30load_word_unaligned:
 31	ldub	[r_TMP + 0x0], r_OFF
 32	ldub	[r_TMP + 0x1], r_TMP2
 33	sll	r_OFF, 8, r_OFF
 34	or	r_OFF, r_TMP2, r_OFF
 35	ldub	[r_TMP + 0x2], r_TMP2
 36	sll	r_OFF, 8, r_OFF
 37	or	r_OFF, r_TMP2, r_OFF
 38	ldub	[r_TMP + 0x3], r_TMP2
 39	sll	r_OFF, 8, r_OFF
 40	retl
 41	 or	r_OFF, r_TMP2, r_RESULT
 42
 43	.globl	bpf_jit_load_half
 44bpf_jit_load_half:
 45	cmp	r_OFF, 0
 46	bl	bpf_slow_path_half_neg
 47	 nop
 48	.globl	bpf_jit_load_half_positive_offset
 49bpf_jit_load_half_positive_offset:
 50	sub	r_HEADLEN, r_OFF, r_TMP
 51	cmp	r_TMP, 1
 52	ble	bpf_slow_path_half
 53	 add	r_SKB_DATA, r_OFF, r_TMP
 54	andcc	r_TMP, 1, %g0
 55	bne	load_half_unaligned
 56	 nop
 57	retl
 58	 lduh	[r_TMP], r_RESULT
 59load_half_unaligned:
 60	ldub	[r_TMP + 0x0], r_OFF
 61	ldub	[r_TMP + 0x1], r_TMP2
 62	sll	r_OFF, 8, r_OFF
 63	retl
 64	 or	r_OFF, r_TMP2, r_RESULT
 65
 66	.globl	bpf_jit_load_byte
 67bpf_jit_load_byte:
 68	cmp	r_OFF, 0
 69	bl	bpf_slow_path_byte_neg
 70	 nop
 71	.globl	bpf_jit_load_byte_positive_offset
 72bpf_jit_load_byte_positive_offset:
 73	cmp	r_OFF, r_HEADLEN
 74	bge	bpf_slow_path_byte
 75	 nop
 76	retl
 77	 ldub	[r_SKB_DATA + r_OFF], r_RESULT
 78
 79#define bpf_slow_path_common(LEN)	\
 80	save	%sp, -SAVE_SZ, %sp;	\
 81	mov	%i0, %o0;		\
 82	mov	%i1, %o1;		\
 83	add	%fp, SCRATCH_OFF, %o2;	\
 84	call	skb_copy_bits;		\
 85	 mov	(LEN), %o3;		\
 86	cmp	%o0, 0;			\
 87	restore;
 88
 89bpf_slow_path_word:
 90	bpf_slow_path_common(4)
 91	bl	bpf_error
 92	 ld	[%sp + SCRATCH_OFF], r_RESULT
 93	retl
 94	 nop
 95bpf_slow_path_half:
 96	bpf_slow_path_common(2)
 97	bl	bpf_error
 98	 lduh	[%sp + SCRATCH_OFF], r_RESULT
 99	retl
100	 nop
101bpf_slow_path_byte:
102	bpf_slow_path_common(1)
103	bl	bpf_error
104	 ldub	[%sp + SCRATCH_OFF], r_RESULT
105	retl
106	 nop
107
108#define bpf_negative_common(LEN)			\
109	save	%sp, -SAVE_SZ, %sp;			\
110	mov	%i0, %o0;				\
111	mov	%i1, %o1;				\
112	SIGN_EXTEND(%o1);				\
113	call	bpf_internal_load_pointer_neg_helper;	\
114	 mov	(LEN), %o2;				\
115	mov	%o0, r_TMP;				\
116	cmp	%o0, 0;					\
117	BE_PTR(bpf_error);				\
118	 restore;
119
120bpf_slow_path_word_neg:
121	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
122	cmp	r_OFF, r_TMP
123	bl	bpf_error
124	 nop
125	.globl	bpf_jit_load_word_negative_offset
126bpf_jit_load_word_negative_offset:
127	bpf_negative_common(4)
128	andcc	r_TMP, 3, %g0
129	bne	load_word_unaligned
130	 nop
131	retl
132	 ld	[r_TMP], r_RESULT
133
134bpf_slow_path_half_neg:
135	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
136	cmp	r_OFF, r_TMP
137	bl	bpf_error
138	 nop
139	.globl	bpf_jit_load_half_negative_offset
140bpf_jit_load_half_negative_offset:
141	bpf_negative_common(2)
142	andcc	r_TMP, 1, %g0
143	bne	load_half_unaligned
144	 nop
145	retl
146	 lduh	[r_TMP], r_RESULT
147
148bpf_slow_path_byte_neg:
149	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
150	cmp	r_OFF, r_TMP
151	bl	bpf_error
152	 nop
153	.globl	bpf_jit_load_byte_negative_offset
154bpf_jit_load_byte_negative_offset:
155	bpf_negative_common(1)
156	retl
157	 ldub	[r_TMP], r_RESULT
158
159bpf_error:
160	/* Make the JIT program itself return zero. */
161	ret
162	restore	%g0, %g0, %o0