1/* bpf_jit.S : BPF JIT helper functions
  2 *
  3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
  4 *
  5 * This program is free software; you can redistribute it and/or
  6 * modify it under the terms of the GNU General Public License
  7 * as published by the Free Software Foundation; version 2
  8 * of the License.
  9 */
 10#include <linux/linkage.h>
 11#include <asm/frame.h>
 12
 13/*
 14 * Calling convention :
 15 * rbx : skb pointer (callee saved)
 16 * esi : offset of byte(s) to fetch in skb (can be scratched)
 17 * r10 : copy of skb->data
 18 * r9d : hlen = skb->len - skb->data_len
 19 */
 20#define SKBDATA	%r10
 21#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
 22#define MAX_BPF_STACK (512 /* from filter.h */ + \
 23	32 /* space for rbx,r13,r14,r15 */ + \
 24	8 /* space for skb_copy_bits */)
 25
 26#define FUNC(name) \
 27	.globl name; \
 28	.type name, @function; \
 29	name:
 30
 31FUNC(sk_load_word)
 32	test	%esi,%esi
 33	js	bpf_slow_path_word_neg
 34
 35FUNC(sk_load_word_positive_offset)
 36	mov	%r9d,%eax		# hlen
 37	sub	%esi,%eax		# hlen - offset
 38	cmp	$3,%eax
 39	jle	bpf_slow_path_word
 40	mov     (SKBDATA,%rsi),%eax
 41	bswap   %eax  			/* ntohl() */
 42	ret
 43
 44FUNC(sk_load_half)
 45	test	%esi,%esi
 46	js	bpf_slow_path_half_neg
 47
 48FUNC(sk_load_half_positive_offset)
 49	mov	%r9d,%eax
 50	sub	%esi,%eax		#	hlen - offset
 51	cmp	$1,%eax
 52	jle	bpf_slow_path_half
 53	movzwl	(SKBDATA,%rsi),%eax
 54	rol	$8,%ax			# ntohs()
 55	ret
 56
 57FUNC(sk_load_byte)
 58	test	%esi,%esi
 59	js	bpf_slow_path_byte_neg
 60
 61FUNC(sk_load_byte_positive_offset)
 62	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
 63	jle	bpf_slow_path_byte
 64	movzbl	(SKBDATA,%rsi),%eax
 65	ret
 66
 67/* rsi contains offset and can be scratched */
 68#define bpf_slow_path_common(LEN)		\
 69	lea	-MAX_BPF_STACK + 32(%rbp), %rdx;\
 70	FRAME_BEGIN;				\
 71	mov	%rbx, %rdi; /* arg1 == skb */	\
 72	push	%r9;				\
 73	push	SKBDATA;			\
 74/* rsi already has offset */			\
 75	mov	$LEN,%ecx;	/* len */	\
 76	call	skb_copy_bits;			\
 77	test    %eax,%eax;			\
 78	pop	SKBDATA;			\
 79	pop	%r9;				\
 80	FRAME_END
 81
 82
 83bpf_slow_path_word:
 84	bpf_slow_path_common(4)
 85	js	bpf_error
 86	mov	- MAX_BPF_STACK + 32(%rbp),%eax
 87	bswap	%eax
 88	ret
 89
 90bpf_slow_path_half:
 91	bpf_slow_path_common(2)
 92	js	bpf_error
 93	mov	- MAX_BPF_STACK + 32(%rbp),%ax
 94	rol	$8,%ax
 95	movzwl	%ax,%eax
 96	ret
 97
 98bpf_slow_path_byte:
 99	bpf_slow_path_common(1)
100	js	bpf_error
101	movzbl	- MAX_BPF_STACK + 32(%rbp),%eax
102	ret
103
104#define sk_negative_common(SIZE)				\
105	FRAME_BEGIN;						\
106	mov	%rbx, %rdi; /* arg1 == skb */			\
107	push	%r9;						\
108	push	SKBDATA;					\
109/* rsi already has offset */					\
110	mov	$SIZE,%edx;	/* size */			\
111	call	bpf_internal_load_pointer_neg_helper;		\
112	test	%rax,%rax;					\
113	pop	SKBDATA;					\
114	pop	%r9;						\
115	FRAME_END;						\
116	jz	bpf_error
117
118bpf_slow_path_word_neg:
119	cmp	SKF_MAX_NEG_OFF, %esi	/* test range */
120	jl	bpf_error	/* offset lower -> error  */
121
122FUNC(sk_load_word_negative_offset)
123	sk_negative_common(4)
124	mov	(%rax), %eax
125	bswap	%eax
126	ret
127
128bpf_slow_path_half_neg:
129	cmp	SKF_MAX_NEG_OFF, %esi
130	jl	bpf_error
131
132FUNC(sk_load_half_negative_offset)
133	sk_negative_common(2)
134	mov	(%rax),%ax
135	rol	$8,%ax
136	movzwl	%ax,%eax
137	ret
138
139bpf_slow_path_byte_neg:
140	cmp	SKF_MAX_NEG_OFF, %esi
141	jl	bpf_error
142
143FUNC(sk_load_byte_negative_offset)
144	sk_negative_common(1)
145	movzbl	(%rax), %eax
146	ret
147
148bpf_error:
149# force a return 0 from jit handler
150	xor	%eax,%eax
151	mov	- MAX_BPF_STACK(%rbp),%rbx
152	mov	- MAX_BPF_STACK + 8(%rbp),%r13
153	mov	- MAX_BPF_STACK + 16(%rbp),%r14
154	mov	- MAX_BPF_STACK + 24(%rbp),%r15
155	leaveq
156	ret