Loading...
1/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
11#include <asm/frame.h>
12
13/*
14 * Calling convention :
15 * rbx : skb pointer (callee saved)
16 * esi : offset of byte(s) to fetch in skb (can be scratched)
17 * r10 : copy of skb->data
18 * r9d : hlen = skb->len - skb->data_len
19 */
20#define SKBDATA %r10
21#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
22
23#define FUNC(name) \
24 .globl name; \
25 .type name, @function; \
26 name:
27
28FUNC(sk_load_word)
29 test %esi,%esi
30 js bpf_slow_path_word_neg
31
32FUNC(sk_load_word_positive_offset)
33 mov %r9d,%eax # hlen
34 sub %esi,%eax # hlen - offset
35 cmp $3,%eax
36 jle bpf_slow_path_word
37 mov (SKBDATA,%rsi),%eax
38 bswap %eax /* ntohl() */
39 ret
40
41FUNC(sk_load_half)
42 test %esi,%esi
43 js bpf_slow_path_half_neg
44
45FUNC(sk_load_half_positive_offset)
46 mov %r9d,%eax
47 sub %esi,%eax # hlen - offset
48 cmp $1,%eax
49 jle bpf_slow_path_half
50 movzwl (SKBDATA,%rsi),%eax
51 rol $8,%ax # ntohs()
52 ret
53
54FUNC(sk_load_byte)
55 test %esi,%esi
56 js bpf_slow_path_byte_neg
57
58FUNC(sk_load_byte_positive_offset)
59 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
60 jle bpf_slow_path_byte
61 movzbl (SKBDATA,%rsi),%eax
62 ret
63
64/* rsi contains offset and can be scratched */
65#define bpf_slow_path_common(LEN) \
66 lea 32(%rbp), %rdx;\
67 FRAME_BEGIN; \
68 mov %rbx, %rdi; /* arg1 == skb */ \
69 push %r9; \
70 push SKBDATA; \
71/* rsi already has offset */ \
72 mov $LEN,%ecx; /* len */ \
73 call skb_copy_bits; \
74 test %eax,%eax; \
75 pop SKBDATA; \
76 pop %r9; \
77 FRAME_END
78
79
80bpf_slow_path_word:
81 bpf_slow_path_common(4)
82 js bpf_error
83 mov 32(%rbp),%eax
84 bswap %eax
85 ret
86
87bpf_slow_path_half:
88 bpf_slow_path_common(2)
89 js bpf_error
90 mov 32(%rbp),%ax
91 rol $8,%ax
92 movzwl %ax,%eax
93 ret
94
95bpf_slow_path_byte:
96 bpf_slow_path_common(1)
97 js bpf_error
98 movzbl 32(%rbp),%eax
99 ret
100
101#define sk_negative_common(SIZE) \
102 FRAME_BEGIN; \
103 mov %rbx, %rdi; /* arg1 == skb */ \
104 push %r9; \
105 push SKBDATA; \
106/* rsi already has offset */ \
107 mov $SIZE,%edx; /* size */ \
108 call bpf_internal_load_pointer_neg_helper; \
109 test %rax,%rax; \
110 pop SKBDATA; \
111 pop %r9; \
112 FRAME_END; \
113 jz bpf_error
114
115bpf_slow_path_word_neg:
116 cmp SKF_MAX_NEG_OFF, %esi /* test range */
117 jl bpf_error /* offset lower -> error */
118
119FUNC(sk_load_word_negative_offset)
120 sk_negative_common(4)
121 mov (%rax), %eax
122 bswap %eax
123 ret
124
125bpf_slow_path_half_neg:
126 cmp SKF_MAX_NEG_OFF, %esi
127 jl bpf_error
128
129FUNC(sk_load_half_negative_offset)
130 sk_negative_common(2)
131 mov (%rax),%ax
132 rol $8,%ax
133 movzwl %ax,%eax
134 ret
135
136bpf_slow_path_byte_neg:
137 cmp SKF_MAX_NEG_OFF, %esi
138 jl bpf_error
139
140FUNC(sk_load_byte_negative_offset)
141 sk_negative_common(1)
142 movzbl (%rax), %eax
143 ret
144
145bpf_error:
146# force a return 0 from jit handler
147 xor %eax,%eax
148 mov (%rbp),%rbx
149 mov 8(%rbp),%r13
150 mov 16(%rbp),%r14
151 mov 24(%rbp),%r15
152 add $40, %rbp
153 leaveq
154 ret
1/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
11#include <asm/dwarf2.h>
12
13/*
14 * Calling convention :
15 * rdi : skb pointer
16 * esi : offset of byte(s) to fetch in skb (can be scratched)
17 * r8 : copy of skb->data
18 * r9d : hlen = skb->len - skb->data_len
19 */
20#define SKBDATA %r8
21#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
22
23sk_load_word:
24 .globl sk_load_word
25
26 test %esi,%esi
27 js bpf_slow_path_word_neg
28
29sk_load_word_positive_offset:
30 .globl sk_load_word_positive_offset
31
32 mov %r9d,%eax # hlen
33 sub %esi,%eax # hlen - offset
34 cmp $3,%eax
35 jle bpf_slow_path_word
36 mov (SKBDATA,%rsi),%eax
37 bswap %eax /* ntohl() */
38 ret
39
40sk_load_half:
41 .globl sk_load_half
42
43 test %esi,%esi
44 js bpf_slow_path_half_neg
45
46sk_load_half_positive_offset:
47 .globl sk_load_half_positive_offset
48
49 mov %r9d,%eax
50 sub %esi,%eax # hlen - offset
51 cmp $1,%eax
52 jle bpf_slow_path_half
53 movzwl (SKBDATA,%rsi),%eax
54 rol $8,%ax # ntohs()
55 ret
56
57sk_load_byte:
58 .globl sk_load_byte
59
60 test %esi,%esi
61 js bpf_slow_path_byte_neg
62
63sk_load_byte_positive_offset:
64 .globl sk_load_byte_positive_offset
65
66 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
67 jle bpf_slow_path_byte
68 movzbl (SKBDATA,%rsi),%eax
69 ret
70
71/**
72 * sk_load_byte_msh - BPF_S_LDX_B_MSH helper
73 *
74 * Implements BPF_S_LDX_B_MSH : ldxb 4*([offset]&0xf)
75 * Must preserve A accumulator (%eax)
76 * Inputs : %esi is the offset value
77 */
78sk_load_byte_msh:
79 .globl sk_load_byte_msh
80 test %esi,%esi
81 js bpf_slow_path_byte_msh_neg
82
83sk_load_byte_msh_positive_offset:
84 .globl sk_load_byte_msh_positive_offset
85 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
86 jle bpf_slow_path_byte_msh
87 movzbl (SKBDATA,%rsi),%ebx
88 and $15,%bl
89 shl $2,%bl
90 ret
91
92/* rsi contains offset and can be scratched */
93#define bpf_slow_path_common(LEN) \
94 push %rdi; /* save skb */ \
95 push %r9; \
96 push SKBDATA; \
97/* rsi already has offset */ \
98 mov $LEN,%ecx; /* len */ \
99 lea -12(%rbp),%rdx; \
100 call skb_copy_bits; \
101 test %eax,%eax; \
102 pop SKBDATA; \
103 pop %r9; \
104 pop %rdi
105
106
107bpf_slow_path_word:
108 bpf_slow_path_common(4)
109 js bpf_error
110 mov -12(%rbp),%eax
111 bswap %eax
112 ret
113
114bpf_slow_path_half:
115 bpf_slow_path_common(2)
116 js bpf_error
117 mov -12(%rbp),%ax
118 rol $8,%ax
119 movzwl %ax,%eax
120 ret
121
122bpf_slow_path_byte:
123 bpf_slow_path_common(1)
124 js bpf_error
125 movzbl -12(%rbp),%eax
126 ret
127
128bpf_slow_path_byte_msh:
129 xchg %eax,%ebx /* dont lose A , X is about to be scratched */
130 bpf_slow_path_common(1)
131 js bpf_error
132 movzbl -12(%rbp),%eax
133 and $15,%al
134 shl $2,%al
135 xchg %eax,%ebx
136 ret
137
138#define sk_negative_common(SIZE) \
139 push %rdi; /* save skb */ \
140 push %r9; \
141 push SKBDATA; \
142/* rsi already has offset */ \
143 mov $SIZE,%edx; /* size */ \
144 call bpf_internal_load_pointer_neg_helper; \
145 test %rax,%rax; \
146 pop SKBDATA; \
147 pop %r9; \
148 pop %rdi; \
149 jz bpf_error
150
151
152bpf_slow_path_word_neg:
153 cmp SKF_MAX_NEG_OFF, %esi /* test range */
154 jl bpf_error /* offset lower -> error */
155sk_load_word_negative_offset:
156 .globl sk_load_word_negative_offset
157 sk_negative_common(4)
158 mov (%rax), %eax
159 bswap %eax
160 ret
161
162bpf_slow_path_half_neg:
163 cmp SKF_MAX_NEG_OFF, %esi
164 jl bpf_error
165sk_load_half_negative_offset:
166 .globl sk_load_half_negative_offset
167 sk_negative_common(2)
168 mov (%rax),%ax
169 rol $8,%ax
170 movzwl %ax,%eax
171 ret
172
173bpf_slow_path_byte_neg:
174 cmp SKF_MAX_NEG_OFF, %esi
175 jl bpf_error
176sk_load_byte_negative_offset:
177 .globl sk_load_byte_negative_offset
178 sk_negative_common(1)
179 movzbl (%rax), %eax
180 ret
181
182bpf_slow_path_byte_msh_neg:
183 cmp SKF_MAX_NEG_OFF, %esi
184 jl bpf_error
185sk_load_byte_msh_negative_offset:
186 .globl sk_load_byte_msh_negative_offset
187 xchg %eax,%ebx /* dont lose A , X is about to be scratched */
188 sk_negative_common(1)
189 movzbl (%rax),%eax
190 and $15,%al
191 shl $2,%al
192 xchg %eax,%ebx
193 ret
194
195bpf_error:
196# force a return 0 from jit handler
197 xor %eax,%eax
198 mov -8(%rbp),%rbx
199 leaveq
200 ret