1/* bpf_jit.S: Packet/header access helper functions
  2 * for PPC64 BPF compiler.
  3 *
  4 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
  5 *
  6 * This program is free software; you can redistribute it and/or
  7 * modify it under the terms of the GNU General Public License
  8 * as published by the Free Software Foundation; version 2
  9 * of the License.
 10 */
 11
 12#include <asm/ppc_asm.h>
 13#include "bpf_jit32.h"
 14
 15/*
 16 * All of these routines are called directly from generated code,
 17 * whose register usage is:
 18 *
 19 * r3		skb
 20 * r4,r5	A,X
 21 * r6		*** address parameter to helper ***
 22 * r7-r10	scratch
 23 * r14		skb->data
 24 * r15		skb headlen
 25 * r16-31	M[]
 26 */
 27
 28/*
 29 * To consider: These helpers are so small it could be better to just
 30 * generate them inline.  Inline code can do the simple headlen check
 31 * then branch directly to slow_path_XXX if required.  (In fact, could
 32 * load a spare GPR with the address of slow_path_generic and pass size
 33 * as an argument, making the call site a mtlr, li and bllr.)
 34 */
 35	.globl	sk_load_word
 36sk_load_word:
 37	PPC_LCMPI	r_addr, 0
 38	blt	bpf_slow_path_word_neg
 39	.globl	sk_load_word_positive_offset
 40sk_load_word_positive_offset:
 41	/* Are we accessing past headlen? */
 42	subi	r_scratch1, r_HL, 4
 43	PPC_LCMP	r_scratch1, r_addr
 44	blt	bpf_slow_path_word
 45	/* Nope, just hitting the header.  cr0 here is eq or gt! */
 46#ifdef __LITTLE_ENDIAN__
 47	lwbrx	r_A, r_D, r_addr
 48#else
 49	lwzx	r_A, r_D, r_addr
 50#endif
 51	blr	/* Return success, cr0 != LT */
 52
 53	.globl	sk_load_half
 54sk_load_half:
 55	PPC_LCMPI	r_addr, 0
 56	blt	bpf_slow_path_half_neg
 57	.globl	sk_load_half_positive_offset
 58sk_load_half_positive_offset:
 59	subi	r_scratch1, r_HL, 2
 60	PPC_LCMP	r_scratch1, r_addr
 61	blt	bpf_slow_path_half
 62#ifdef __LITTLE_ENDIAN__
 63	lhbrx	r_A, r_D, r_addr
 64#else
 65	lhzx	r_A, r_D, r_addr
 66#endif
 67	blr
 68
 69	.globl	sk_load_byte
 70sk_load_byte:
 71	PPC_LCMPI	r_addr, 0
 72	blt	bpf_slow_path_byte_neg
 73	.globl	sk_load_byte_positive_offset
 74sk_load_byte_positive_offset:
 75	PPC_LCMP	r_HL, r_addr
 76	ble	bpf_slow_path_byte
 77	lbzx	r_A, r_D, r_addr
 78	blr
 79
 80/*
 81 * BPF_LDX | BPF_B | BPF_MSH: ldxb  4*([offset]&0xf)
 82 * r_addr is the offset value
 83 */
 84	.globl sk_load_byte_msh
 85sk_load_byte_msh:
 86	PPC_LCMPI	r_addr, 0
 87	blt	bpf_slow_path_byte_msh_neg
 88	.globl sk_load_byte_msh_positive_offset
 89sk_load_byte_msh_positive_offset:
 90	PPC_LCMP	r_HL, r_addr
 91	ble	bpf_slow_path_byte_msh
 92	lbzx	r_X, r_D, r_addr
 93	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
 94	blr
 95
 96/* Call out to skb_copy_bits:
 97 * We'll need to back up our volatile regs first; we have
 98 * local variable space at r1+(BPF_PPC_STACK_BASIC).
 99 * Allocate a new stack frame here to remain ABI-compliant in
100 * stashing LR.
101 */
102#define bpf_slow_path_common(SIZE)				\
103	mflr	r0;						\
104	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
105	/* R3 goes in parameter space of caller's frame */	\
106	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
107	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
108	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
109	addi	r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ);		\
110	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
111	/* R3 = r_skb, as passed */				\
112	mr	r4, r_addr;					\
113	li	r6, SIZE;					\
114	bl	skb_copy_bits;					\
115	nop;							\
116	/* R3 = 0 on success */					\
117	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
118	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
119	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
120	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
121	mtlr	r0;						\
122	PPC_LCMPI	r3, 0;						\
123	blt	bpf_error;	/* cr0 = LT */			\
124	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
125	/* Great success! */
126
127bpf_slow_path_word:
128	bpf_slow_path_common(4)
129	/* Data value is on stack, and cr0 != LT */
130	lwz	r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
131	blr
132
133bpf_slow_path_half:
134	bpf_slow_path_common(2)
135	lhz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
136	blr
137
138bpf_slow_path_byte:
139	bpf_slow_path_common(1)
140	lbz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
141	blr
142
143bpf_slow_path_byte_msh:
144	bpf_slow_path_common(1)
145	lbz	r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
146	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
147	blr
148
149/* Call out to bpf_internal_load_pointer_neg_helper:
150 * We'll need to back up our volatile regs first; we have
151 * local variable space at r1+(BPF_PPC_STACK_BASIC).
152 * Allocate a new stack frame here to remain ABI-compliant in
153 * stashing LR.
154 */
155#define sk_negative_common(SIZE)				\
156	mflr	r0;						\
157	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
158	/* R3 goes in parameter space of caller's frame */	\
159	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
160	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
161	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
162	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
163	/* R3 = r_skb, as passed */				\
164	mr	r4, r_addr;					\
165	li	r5, SIZE;					\
166	bl	bpf_internal_load_pointer_neg_helper;		\
167	nop;							\
168	/* R3 != 0 on success */				\
169	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
170	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
171	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
172	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
173	mtlr	r0;						\
174	PPC_LCMPLI	r3, 0;						\
175	beq	bpf_error_slow;	/* cr0 = EQ */			\
176	mr	r_addr, r3;					\
177	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
178	/* Great success! */
179
180bpf_slow_path_word_neg:
181	lis     r_scratch1,-32	/* SKF_LL_OFF */
182	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
183	blt	bpf_error	/* cr0 = LT */
184	.globl	sk_load_word_negative_offset
185sk_load_word_negative_offset:
186	sk_negative_common(4)
187	lwz	r_A, 0(r_addr)
188	blr
189
190bpf_slow_path_half_neg:
191	lis     r_scratch1,-32	/* SKF_LL_OFF */
192	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
193	blt	bpf_error	/* cr0 = LT */
194	.globl	sk_load_half_negative_offset
195sk_load_half_negative_offset:
196	sk_negative_common(2)
197	lhz	r_A, 0(r_addr)
198	blr
199
200bpf_slow_path_byte_neg:
201	lis     r_scratch1,-32	/* SKF_LL_OFF */
202	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
203	blt	bpf_error	/* cr0 = LT */
204	.globl	sk_load_byte_negative_offset
205sk_load_byte_negative_offset:
206	sk_negative_common(1)
207	lbz	r_A, 0(r_addr)
208	blr
209
210bpf_slow_path_byte_msh_neg:
211	lis     r_scratch1,-32	/* SKF_LL_OFF */
212	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
213	blt	bpf_error	/* cr0 = LT */
214	.globl	sk_load_byte_msh_negative_offset
215sk_load_byte_msh_negative_offset:
216	sk_negative_common(1)
217	lbz	r_X, 0(r_addr)
218	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
219	blr
220
221bpf_error_slow:
222	/* fabricate a cr0 = lt */
223	li	r_scratch1, -1
224	PPC_LCMPI	r_scratch1, 0
225bpf_error:
226	/* Entered with cr0 = lt */
227	li	r3, 0
228	/* Generated code will 'blt epilogue', returning 0. */
229	blr