1/* bpf_jit.S : BPF JIT helper functions
  2 *
  3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
  4 *
  5 * This program is free software; you can redistribute it and/or
  6 * modify it under the terms of the GNU General Public License
  7 * as published by the Free Software Foundation; version 2
  8 * of the License.
  9 */
 10#include <linux/linkage.h>
 11#include <asm/frame.h>
 12
 13/*
 14 * Calling convention :
 15 * rbx : skb pointer (callee saved)
 16 * esi : offset of byte(s) to fetch in skb (can be scratched)
 17 * r10 : copy of skb->data
 18 * r9d : hlen = skb->len - skb->data_len
 19 */
 20#define SKBDATA	%r10
 21#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
 22
 23#define FUNC(name) \
 24	.globl name; \
 25	.type name, @function; \
 26	name:
 27
 28FUNC(sk_load_word)
 29	test	%esi,%esi
 30	js	bpf_slow_path_word_neg
 31
 32FUNC(sk_load_word_positive_offset)
 33	mov	%r9d,%eax		# hlen
 34	sub	%esi,%eax		# hlen - offset
 35	cmp	$3,%eax
 36	jle	bpf_slow_path_word
 37	mov     (SKBDATA,%rsi),%eax
 38	bswap   %eax  			/* ntohl() */
 39	ret
 40
 41FUNC(sk_load_half)
 42	test	%esi,%esi
 43	js	bpf_slow_path_half_neg
 44
 45FUNC(sk_load_half_positive_offset)
 46	mov	%r9d,%eax
 47	sub	%esi,%eax		#	hlen - offset
 48	cmp	$1,%eax
 49	jle	bpf_slow_path_half
 50	movzwl	(SKBDATA,%rsi),%eax
 51	rol	$8,%ax			# ntohs()
 52	ret
 53
 54FUNC(sk_load_byte)
 55	test	%esi,%esi
 56	js	bpf_slow_path_byte_neg
 57
 58FUNC(sk_load_byte_positive_offset)
 59	cmp	%esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
 60	jle	bpf_slow_path_byte
 61	movzbl	(SKBDATA,%rsi),%eax
 62	ret
 63
 64/* rsi contains offset and can be scratched */
 65#define bpf_slow_path_common(LEN)		\
 66	lea	32(%rbp), %rdx;\
 67	FRAME_BEGIN;				\
 68	mov	%rbx, %rdi; /* arg1 == skb */	\
 69	push	%r9;				\
 70	push	SKBDATA;			\
 71/* rsi already has offset */			\
 72	mov	$LEN,%ecx;	/* len */	\
 73	call	skb_copy_bits;			\
 74	test    %eax,%eax;			\
 75	pop	SKBDATA;			\
 76	pop	%r9;				\
 77	FRAME_END
 78
 79
 80bpf_slow_path_word:
 81	bpf_slow_path_common(4)
 82	js	bpf_error
 83	mov	32(%rbp),%eax
 84	bswap	%eax
 85	ret
 86
 87bpf_slow_path_half:
 88	bpf_slow_path_common(2)
 89	js	bpf_error
 90	mov	32(%rbp),%ax
 91	rol	$8,%ax
 92	movzwl	%ax,%eax
 93	ret
 94
 95bpf_slow_path_byte:
 96	bpf_slow_path_common(1)
 97	js	bpf_error
 98	movzbl	32(%rbp),%eax
 99	ret
100
101#define sk_negative_common(SIZE)				\
102	FRAME_BEGIN;						\
103	mov	%rbx, %rdi; /* arg1 == skb */			\
104	push	%r9;						\
105	push	SKBDATA;					\
106/* rsi already has offset */					\
107	mov	$SIZE,%edx;	/* size */			\
108	call	bpf_internal_load_pointer_neg_helper;		\
109	test	%rax,%rax;					\
110	pop	SKBDATA;					\
111	pop	%r9;						\
112	FRAME_END;						\
113	jz	bpf_error
114
115bpf_slow_path_word_neg:
116	cmp	SKF_MAX_NEG_OFF, %esi	/* test range */
117	jl	bpf_error	/* offset lower -> error  */
118
119FUNC(sk_load_word_negative_offset)
120	sk_negative_common(4)
121	mov	(%rax), %eax
122	bswap	%eax
123	ret
124
125bpf_slow_path_half_neg:
126	cmp	SKF_MAX_NEG_OFF, %esi
127	jl	bpf_error
128
129FUNC(sk_load_half_negative_offset)
130	sk_negative_common(2)
131	mov	(%rax),%ax
132	rol	$8,%ax
133	movzwl	%ax,%eax
134	ret
135
136bpf_slow_path_byte_neg:
137	cmp	SKF_MAX_NEG_OFF, %esi
138	jl	bpf_error
139
140FUNC(sk_load_byte_negative_offset)
141	sk_negative_common(1)
142	movzbl	(%rax), %eax
143	ret
144
145bpf_error:
146# force a return 0 from jit handler
147	xor	%eax,%eax
148	mov	(%rbp),%rbx
149	mov	8(%rbp),%r13
150	mov	16(%rbp),%r14
151	mov	24(%rbp),%r15
152	add	$40, %rbp
153	leaveq
154	ret