Loading...
1/*
2 * Just-In-Time compiler for eBPF filters on 32bit ARM
3 *
4 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
5 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
10 */
11
12#include <linux/bpf.h>
13#include <linux/bitops.h>
14#include <linux/compiler.h>
15#include <linux/errno.h>
16#include <linux/filter.h>
17#include <linux/netdevice.h>
18#include <linux/string.h>
19#include <linux/slab.h>
20#include <linux/if_vlan.h>
21
22#include <asm/cacheflush.h>
23#include <asm/hwcap.h>
24#include <asm/opcodes.h>
25
26#include "bpf_jit_32.h"
27
28/*
29 * eBPF prog stack layout:
30 *
31 * high
32 * original ARM_SP => +-----+
33 * | | callee saved registers
34 * +-----+ <= (BPF_FP + SCRATCH_SIZE)
35 * | ... | eBPF JIT scratch space
36 * eBPF fp register => +-----+
37 * (BPF_FP) | ... | eBPF prog stack
38 * +-----+
39 * |RSVD | JIT scratchpad
40 * current ARM_SP => +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE)
41 * | |
42 * | ... | Function call stack
43 * | |
44 * +-----+
45 * low
46 *
47 * The callee saved registers depends on whether frame pointers are enabled.
48 * With frame pointers (to be compliant with the ABI):
49 *
50 * high
51 * original ARM_SP => +------------------+ \
52 * | pc | |
53 * current ARM_FP => +------------------+ } callee saved registers
54 * |r4-r8,r10,fp,ip,lr| |
55 * +------------------+ /
56 * low
57 *
58 * Without frame pointers:
59 *
60 * high
61 * original ARM_SP => +------------------+
62 * | r4-r8,r10,fp,lr | callee saved registers
63 * current ARM_FP => +------------------+
64 * low
65 *
66 * When popping registers off the stack at the end of a BPF function, we
67 * reference them via the current ARM_FP register.
68 */
69#define CALLEE_MASK (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \
70 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R10 | \
71 1 << ARM_FP)
72#define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR)
73#define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC)
74
75#define STACK_OFFSET(k) (k)
76#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */
77#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */
78#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */
79
80#define FLAG_IMM_OVERFLOW (1 << 0)
81
82/*
83 * Map eBPF registers to ARM 32bit registers or stack scratch space.
84 *
85 * 1. First argument is passed using the arm 32bit registers and rest of the
86 * arguments are passed on stack scratch space.
87 * 2. First callee-saved arugument is mapped to arm 32 bit registers and rest
88 * arguments are mapped to scratch space on stack.
89 * 3. We need two 64 bit temp registers to do complex operations on eBPF
90 * registers.
91 *
92 * As the eBPF registers are all 64 bit registers and arm has only 32 bit
93 * registers, we have to map each eBPF registers with two arm 32 bit regs or
94 * scratch memory space and we have to build eBPF 64 bit register from those.
95 *
96 */
97static const u8 bpf2a32[][2] = {
98 /* return value from in-kernel function, and exit value from eBPF */
99 [BPF_REG_0] = {ARM_R1, ARM_R0},
100 /* arguments from eBPF program to in-kernel function */
101 [BPF_REG_1] = {ARM_R3, ARM_R2},
102 /* Stored on stack scratch space */
103 [BPF_REG_2] = {STACK_OFFSET(0), STACK_OFFSET(4)},
104 [BPF_REG_3] = {STACK_OFFSET(8), STACK_OFFSET(12)},
105 [BPF_REG_4] = {STACK_OFFSET(16), STACK_OFFSET(20)},
106 [BPF_REG_5] = {STACK_OFFSET(24), STACK_OFFSET(28)},
107 /* callee saved registers that in-kernel function will preserve */
108 [BPF_REG_6] = {ARM_R5, ARM_R4},
109 /* Stored on stack scratch space */
110 [BPF_REG_7] = {STACK_OFFSET(32), STACK_OFFSET(36)},
111 [BPF_REG_8] = {STACK_OFFSET(40), STACK_OFFSET(44)},
112 [BPF_REG_9] = {STACK_OFFSET(48), STACK_OFFSET(52)},
113 /* Read only Frame Pointer to access Stack */
114 [BPF_REG_FP] = {STACK_OFFSET(56), STACK_OFFSET(60)},
115 /* Temporary Register for internal BPF JIT, can be used
116 * for constant blindings and others.
117 */
118 [TMP_REG_1] = {ARM_R7, ARM_R6},
119 [TMP_REG_2] = {ARM_R10, ARM_R8},
120 /* Tail call count. Stored on stack scratch space. */
121 [TCALL_CNT] = {STACK_OFFSET(64), STACK_OFFSET(68)},
122 /* temporary register for blinding constants.
123 * Stored on stack scratch space.
124 */
125 [BPF_REG_AX] = {STACK_OFFSET(72), STACK_OFFSET(76)},
126};
127
128#define dst_lo dst[1]
129#define dst_hi dst[0]
130#define src_lo src[1]
131#define src_hi src[0]
132
133/*
134 * JIT Context:
135 *
136 * prog : bpf_prog
137 * idx : index of current last JITed instruction.
138 * prologue_bytes : bytes used in prologue.
139 * epilogue_offset : offset of epilogue starting.
140 * offsets : array of eBPF instruction offsets in
141 * JITed code.
142 * target : final JITed code.
143 * epilogue_bytes : no of bytes used in epilogue.
144 * imm_count : no of immediate counts used for global
145 * variables.
146 * imms : array of global variable addresses.
147 */
148
149struct jit_ctx {
150 const struct bpf_prog *prog;
151 unsigned int idx;
152 unsigned int prologue_bytes;
153 unsigned int epilogue_offset;
154 u32 flags;
155 u32 *offsets;
156 u32 *target;
157 u32 stack_size;
158#if __LINUX_ARM_ARCH__ < 7
159 u16 epilogue_bytes;
160 u16 imm_count;
161 u32 *imms;
162#endif
163};
164
165/*
166 * Wrappers which handle both OABI and EABI and assures Thumb2 interworking
167 * (where the assembly routines like __aeabi_uidiv could cause problems).
168 */
169static u32 jit_udiv32(u32 dividend, u32 divisor)
170{
171 return dividend / divisor;
172}
173
174static u32 jit_mod32(u32 dividend, u32 divisor)
175{
176 return dividend % divisor;
177}
178
179static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
180{
181 inst |= (cond << 28);
182 inst = __opcode_to_mem_arm(inst);
183
184 if (ctx->target != NULL)
185 ctx->target[ctx->idx] = inst;
186
187 ctx->idx++;
188}
189
190/*
191 * Emit an instruction that will be executed unconditionally.
192 */
193static inline void emit(u32 inst, struct jit_ctx *ctx)
194{
195 _emit(ARM_COND_AL, inst, ctx);
196}
197
198/*
199 * Checks if immediate value can be converted to imm12(12 bits) value.
200 */
201static int16_t imm8m(u32 x)
202{
203 u32 rot;
204
205 for (rot = 0; rot < 16; rot++)
206 if ((x & ~ror32(0xff, 2 * rot)) == 0)
207 return rol32(x, 2 * rot) | (rot << 8);
208 return -1;
209}
210
211/*
212 * Initializes the JIT space with undefined instructions.
213 */
214static void jit_fill_hole(void *area, unsigned int size)
215{
216 u32 *ptr;
217 /* We are guaranteed to have aligned memory. */
218 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
219 *ptr++ = __opcode_to_mem_arm(ARM_INST_UDF);
220}
221
222#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
223/* EABI requires the stack to be aligned to 64-bit boundaries */
224#define STACK_ALIGNMENT 8
225#else
226/* Stack must be aligned to 32-bit boundaries */
227#define STACK_ALIGNMENT 4
228#endif
229
230/* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4,
231 * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9,
232 * BPF_REG_FP and Tail call counts.
233 */
234#define SCRATCH_SIZE 80
235
236/* total stack size used in JITed code */
237#define _STACK_SIZE \
238 (ctx->prog->aux->stack_depth + \
239 + SCRATCH_SIZE + \
240 + 4 /* extra for skb_copy_bits buffer */)
241
242#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT)
243
244/* Get the offset of eBPF REGISTERs stored on scratch space. */
245#define STACK_VAR(off) (STACK_SIZE-off-4)
246
247/* Offset of skb_copy_bits buffer */
248#define SKB_BUFFER STACK_VAR(SCRATCH_SIZE)
249
250#if __LINUX_ARM_ARCH__ < 7
251
252static u16 imm_offset(u32 k, struct jit_ctx *ctx)
253{
254 unsigned int i = 0, offset;
255 u16 imm;
256
257 /* on the "fake" run we just count them (duplicates included) */
258 if (ctx->target == NULL) {
259 ctx->imm_count++;
260 return 0;
261 }
262
263 while ((i < ctx->imm_count) && ctx->imms[i]) {
264 if (ctx->imms[i] == k)
265 break;
266 i++;
267 }
268
269 if (ctx->imms[i] == 0)
270 ctx->imms[i] = k;
271
272 /* constants go just after the epilogue */
273 offset = ctx->offsets[ctx->prog->len - 1] * 4;
274 offset += ctx->prologue_bytes;
275 offset += ctx->epilogue_bytes;
276 offset += i * 4;
277
278 ctx->target[offset / 4] = k;
279
280 /* PC in ARM mode == address of the instruction + 8 */
281 imm = offset - (8 + ctx->idx * 4);
282
283 if (imm & ~0xfff) {
284 /*
285 * literal pool is too far, signal it into flags. we
286 * can only detect it on the second pass unfortunately.
287 */
288 ctx->flags |= FLAG_IMM_OVERFLOW;
289 return 0;
290 }
291
292 return imm;
293}
294
295#endif /* __LINUX_ARM_ARCH__ */
296
297static inline int bpf2a32_offset(int bpf_to, int bpf_from,
298 const struct jit_ctx *ctx) {
299 int to, from;
300
301 if (ctx->target == NULL)
302 return 0;
303 to = ctx->offsets[bpf_to];
304 from = ctx->offsets[bpf_from];
305
306 return to - from - 1;
307}
308
309/*
310 * Move an immediate that's not an imm8m to a core register.
311 */
312static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx)
313{
314#if __LINUX_ARM_ARCH__ < 7
315 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);
316#else
317 emit(ARM_MOVW(rd, val & 0xffff), ctx);
318 if (val > 0xffff)
319 emit(ARM_MOVT(rd, val >> 16), ctx);
320#endif
321}
322
323static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx)
324{
325 int imm12 = imm8m(val);
326
327 if (imm12 >= 0)
328 emit(ARM_MOV_I(rd, imm12), ctx);
329 else
330 emit_mov_i_no8m(rd, val, ctx);
331}
332
333static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx)
334{
335 if (elf_hwcap & HWCAP_THUMB)
336 emit(ARM_BX(tgt_reg), ctx);
337 else
338 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);
339}
340
341static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
342{
343#if __LINUX_ARM_ARCH__ < 5
344 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);
345 emit_bx_r(tgt_reg, ctx);
346#else
347 emit(ARM_BLX_R(tgt_reg), ctx);
348#endif
349}
350
351static inline int epilogue_offset(const struct jit_ctx *ctx)
352{
353 int to, from;
354 /* No need for 1st dummy run */
355 if (ctx->target == NULL)
356 return 0;
357 to = ctx->epilogue_offset;
358 from = ctx->idx;
359
360 return to - from - 2;
361}
362
363static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
364{
365 const u8 *tmp = bpf2a32[TMP_REG_1];
366
367#if __LINUX_ARM_ARCH__ == 7
368 if (elf_hwcap & HWCAP_IDIVA) {
369 if (op == BPF_DIV)
370 emit(ARM_UDIV(rd, rm, rn), ctx);
371 else {
372 emit(ARM_UDIV(ARM_IP, rm, rn), ctx);
373 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);
374 }
375 return;
376 }
377#endif
378
379 /*
380 * For BPF_ALU | BPF_DIV | BPF_K instructions
381 * As ARM_R1 and ARM_R0 contains 1st argument of bpf
382 * function, we need to save it on caller side to save
383 * it from getting destroyed within callee.
384 * After the return from the callee, we restore ARM_R0
385 * ARM_R1.
386 */
387 if (rn != ARM_R1) {
388 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);
389 emit(ARM_MOV_R(ARM_R1, rn), ctx);
390 }
391 if (rm != ARM_R0) {
392 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);
393 emit(ARM_MOV_R(ARM_R0, rm), ctx);
394 }
395
396 /* Call appropriate function */
397 emit_mov_i(ARM_IP, op == BPF_DIV ?
398 (u32)jit_udiv32 : (u32)jit_mod32, ctx);
399 emit_blx_r(ARM_IP, ctx);
400
401 /* Save return value */
402 if (rd != ARM_R0)
403 emit(ARM_MOV_R(rd, ARM_R0), ctx);
404
405 /* Restore ARM_R0 and ARM_R1 */
406 if (rn != ARM_R1)
407 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);
408 if (rm != ARM_R0)
409 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);
410}
411
412/* Checks whether BPF register is on scratch stack space or not. */
413static inline bool is_on_stack(u8 bpf_reg)
414{
415 static u8 stack_regs[] = {BPF_REG_AX, BPF_REG_3, BPF_REG_4, BPF_REG_5,
416 BPF_REG_7, BPF_REG_8, BPF_REG_9, TCALL_CNT,
417 BPF_REG_2, BPF_REG_FP};
418 int i, reg_len = sizeof(stack_regs);
419
420 for (i = 0 ; i < reg_len ; i++) {
421 if (bpf_reg == stack_regs[i])
422 return true;
423 }
424 return false;
425}
426
427static inline void emit_a32_mov_i(const u8 dst, const u32 val,
428 bool dstk, struct jit_ctx *ctx)
429{
430 const u8 *tmp = bpf2a32[TMP_REG_1];
431
432 if (dstk) {
433 emit_mov_i(tmp[1], val, ctx);
434 emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(dst)), ctx);
435 } else {
436 emit_mov_i(dst, val, ctx);
437 }
438}
439
440/* Sign extended move */
441static inline void emit_a32_mov_i64(const bool is64, const u8 dst[],
442 const u32 val, bool dstk,
443 struct jit_ctx *ctx) {
444 u32 hi = 0;
445
446 if (is64 && (val & (1<<31)))
447 hi = (u32)~0;
448 emit_a32_mov_i(dst_lo, val, dstk, ctx);
449 emit_a32_mov_i(dst_hi, hi, dstk, ctx);
450}
451
452static inline void emit_a32_add_r(const u8 dst, const u8 src,
453 const bool is64, const bool hi,
454 struct jit_ctx *ctx) {
455 /* 64 bit :
456 * adds dst_lo, dst_lo, src_lo
457 * adc dst_hi, dst_hi, src_hi
458 * 32 bit :
459 * add dst_lo, dst_lo, src_lo
460 */
461 if (!hi && is64)
462 emit(ARM_ADDS_R(dst, dst, src), ctx);
463 else if (hi && is64)
464 emit(ARM_ADC_R(dst, dst, src), ctx);
465 else
466 emit(ARM_ADD_R(dst, dst, src), ctx);
467}
468
469static inline void emit_a32_sub_r(const u8 dst, const u8 src,
470 const bool is64, const bool hi,
471 struct jit_ctx *ctx) {
472 /* 64 bit :
473 * subs dst_lo, dst_lo, src_lo
474 * sbc dst_hi, dst_hi, src_hi
475 * 32 bit :
476 * sub dst_lo, dst_lo, src_lo
477 */
478 if (!hi && is64)
479 emit(ARM_SUBS_R(dst, dst, src), ctx);
480 else if (hi && is64)
481 emit(ARM_SBC_R(dst, dst, src), ctx);
482 else
483 emit(ARM_SUB_R(dst, dst, src), ctx);
484}
485
486static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64,
487 const bool hi, const u8 op, struct jit_ctx *ctx){
488 switch (BPF_OP(op)) {
489 /* dst = dst + src */
490 case BPF_ADD:
491 emit_a32_add_r(dst, src, is64, hi, ctx);
492 break;
493 /* dst = dst - src */
494 case BPF_SUB:
495 emit_a32_sub_r(dst, src, is64, hi, ctx);
496 break;
497 /* dst = dst | src */
498 case BPF_OR:
499 emit(ARM_ORR_R(dst, dst, src), ctx);
500 break;
501 /* dst = dst & src */
502 case BPF_AND:
503 emit(ARM_AND_R(dst, dst, src), ctx);
504 break;
505 /* dst = dst ^ src */
506 case BPF_XOR:
507 emit(ARM_EOR_R(dst, dst, src), ctx);
508 break;
509 /* dst = dst * src */
510 case BPF_MUL:
511 emit(ARM_MUL(dst, dst, src), ctx);
512 break;
513 /* dst = dst << src */
514 case BPF_LSH:
515 emit(ARM_LSL_R(dst, dst, src), ctx);
516 break;
517 /* dst = dst >> src */
518 case BPF_RSH:
519 emit(ARM_LSR_R(dst, dst, src), ctx);
520 break;
521 /* dst = dst >> src (signed)*/
522 case BPF_ARSH:
523 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);
524 break;
525 }
526}
527
528/* ALU operation (32 bit)
529 * dst = dst (op) src
530 */
531static inline void emit_a32_alu_r(const u8 dst, const u8 src,
532 bool dstk, bool sstk,
533 struct jit_ctx *ctx, const bool is64,
534 const bool hi, const u8 op) {
535 const u8 *tmp = bpf2a32[TMP_REG_1];
536 u8 rn = sstk ? tmp[1] : src;
537
538 if (sstk)
539 emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src)), ctx);
540
541 /* ALU operation */
542 if (dstk) {
543 emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
544 emit_alu_r(tmp[0], rn, is64, hi, op, ctx);
545 emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(dst)), ctx);
546 } else {
547 emit_alu_r(dst, rn, is64, hi, op, ctx);
548 }
549}
550
551/* ALU operation (64 bit) */
552static inline void emit_a32_alu_r64(const bool is64, const u8 dst[],
553 const u8 src[], bool dstk,
554 bool sstk, struct jit_ctx *ctx,
555 const u8 op) {
556 emit_a32_alu_r(dst_lo, src_lo, dstk, sstk, ctx, is64, false, op);
557 if (is64)
558 emit_a32_alu_r(dst_hi, src_hi, dstk, sstk, ctx, is64, true, op);
559 else
560 emit_a32_mov_i(dst_hi, 0, dstk, ctx);
561}
562
563/* dst = imm (4 bytes)*/
564static inline void emit_a32_mov_r(const u8 dst, const u8 src,
565 bool dstk, bool sstk,
566 struct jit_ctx *ctx) {
567 const u8 *tmp = bpf2a32[TMP_REG_1];
568 u8 rt = sstk ? tmp[0] : src;
569
570 if (sstk)
571 emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(src)), ctx);
572 if (dstk)
573 emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst)), ctx);
574 else
575 emit(ARM_MOV_R(dst, rt), ctx);
576}
577
578/* dst = src */
579static inline void emit_a32_mov_r64(const bool is64, const u8 dst[],
580 const u8 src[], bool dstk,
581 bool sstk, struct jit_ctx *ctx) {
582 emit_a32_mov_r(dst_lo, src_lo, dstk, sstk, ctx);
583 if (is64) {
584 /* complete 8 byte move */
585 emit_a32_mov_r(dst_hi, src_hi, dstk, sstk, ctx);
586 } else {
587 /* Zero out high 4 bytes */
588 emit_a32_mov_i(dst_hi, 0, dstk, ctx);
589 }
590}
591
592/* Shift operations */
593static inline void emit_a32_alu_i(const u8 dst, const u32 val, bool dstk,
594 struct jit_ctx *ctx, const u8 op) {
595 const u8 *tmp = bpf2a32[TMP_REG_1];
596 u8 rd = dstk ? tmp[0] : dst;
597
598 if (dstk)
599 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
600
601 /* Do shift operation */
602 switch (op) {
603 case BPF_LSH:
604 emit(ARM_LSL_I(rd, rd, val), ctx);
605 break;
606 case BPF_RSH:
607 emit(ARM_LSR_I(rd, rd, val), ctx);
608 break;
609 case BPF_NEG:
610 emit(ARM_RSB_I(rd, rd, val), ctx);
611 break;
612 }
613
614 if (dstk)
615 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
616}
617
618/* dst = ~dst (64 bit) */
619static inline void emit_a32_neg64(const u8 dst[], bool dstk,
620 struct jit_ctx *ctx){
621 const u8 *tmp = bpf2a32[TMP_REG_1];
622 u8 rd = dstk ? tmp[1] : dst[1];
623 u8 rm = dstk ? tmp[0] : dst[0];
624
625 /* Setup Operand */
626 if (dstk) {
627 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
628 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
629 }
630
631 /* Do Negate Operation */
632 emit(ARM_RSBS_I(rd, rd, 0), ctx);
633 emit(ARM_RSC_I(rm, rm, 0), ctx);
634
635 if (dstk) {
636 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
637 emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
638 }
639}
640
641/* dst = dst << src */
642static inline void emit_a32_lsh_r64(const u8 dst[], const u8 src[], bool dstk,
643 bool sstk, struct jit_ctx *ctx) {
644 const u8 *tmp = bpf2a32[TMP_REG_1];
645 const u8 *tmp2 = bpf2a32[TMP_REG_2];
646
647 /* Setup Operands */
648 u8 rt = sstk ? tmp2[1] : src_lo;
649 u8 rd = dstk ? tmp[1] : dst_lo;
650 u8 rm = dstk ? tmp[0] : dst_hi;
651
652 if (sstk)
653 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
654 if (dstk) {
655 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
656 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
657 }
658
659 /* Do LSH operation */
660 emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
661 emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);
662 emit(ARM_MOV_SR(ARM_LR, rm, SRTYPE_ASL, rt), ctx);
663 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd, SRTYPE_ASL, ARM_IP), ctx);
664 emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd, SRTYPE_LSR, tmp2[0]), ctx);
665 emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_ASL, rt), ctx);
666
667 if (dstk) {
668 emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
669 emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
670 } else {
671 emit(ARM_MOV_R(rd, ARM_LR), ctx);
672 emit(ARM_MOV_R(rm, ARM_IP), ctx);
673 }
674}
675
676/* dst = dst >> src (signed)*/
677static inline void emit_a32_arsh_r64(const u8 dst[], const u8 src[], bool dstk,
678 bool sstk, struct jit_ctx *ctx) {
679 const u8 *tmp = bpf2a32[TMP_REG_1];
680 const u8 *tmp2 = bpf2a32[TMP_REG_2];
681 /* Setup Operands */
682 u8 rt = sstk ? tmp2[1] : src_lo;
683 u8 rd = dstk ? tmp[1] : dst_lo;
684 u8 rm = dstk ? tmp[0] : dst_hi;
685
686 if (sstk)
687 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
688 if (dstk) {
689 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
690 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
691 }
692
693 /* Do the ARSH operation */
694 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
695 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
696 emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
697 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
698 _emit(ARM_COND_MI, ARM_B(0), ctx);
699 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASR, tmp2[0]), ctx);
700 emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_ASR, rt), ctx);
701 if (dstk) {
702 emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
703 emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
704 } else {
705 emit(ARM_MOV_R(rd, ARM_LR), ctx);
706 emit(ARM_MOV_R(rm, ARM_IP), ctx);
707 }
708}
709
710/* dst = dst >> src */
711static inline void emit_a32_lsr_r64(const u8 dst[], const u8 src[], bool dstk,
712 bool sstk, struct jit_ctx *ctx) {
713 const u8 *tmp = bpf2a32[TMP_REG_1];
714 const u8 *tmp2 = bpf2a32[TMP_REG_2];
715 /* Setup Operands */
716 u8 rt = sstk ? tmp2[1] : src_lo;
717 u8 rd = dstk ? tmp[1] : dst_lo;
718 u8 rm = dstk ? tmp[0] : dst_hi;
719
720 if (sstk)
721 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
722 if (dstk) {
723 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
724 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
725 }
726
727 /* Do LSH operation */
728 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
729 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
730 emit(ARM_MOV_SR(ARM_LR, rd, SRTYPE_LSR, rt), ctx);
731 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_ASL, ARM_IP), ctx);
732 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rm, SRTYPE_LSR, tmp2[0]), ctx);
733 emit(ARM_MOV_SR(ARM_IP, rm, SRTYPE_LSR, rt), ctx);
734 if (dstk) {
735 emit(ARM_STR_I(ARM_LR, ARM_SP, STACK_VAR(dst_lo)), ctx);
736 emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_hi)), ctx);
737 } else {
738 emit(ARM_MOV_R(rd, ARM_LR), ctx);
739 emit(ARM_MOV_R(rm, ARM_IP), ctx);
740 }
741}
742
743/* dst = dst << val */
744static inline void emit_a32_lsh_i64(const u8 dst[], bool dstk,
745 const u32 val, struct jit_ctx *ctx){
746 const u8 *tmp = bpf2a32[TMP_REG_1];
747 const u8 *tmp2 = bpf2a32[TMP_REG_2];
748 /* Setup operands */
749 u8 rd = dstk ? tmp[1] : dst_lo;
750 u8 rm = dstk ? tmp[0] : dst_hi;
751
752 if (dstk) {
753 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
754 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
755 }
756
757 /* Do LSH operation */
758 if (val < 32) {
759 emit(ARM_MOV_SI(tmp2[0], rm, SRTYPE_ASL, val), ctx);
760 emit(ARM_ORR_SI(rm, tmp2[0], rd, SRTYPE_LSR, 32 - val), ctx);
761 emit(ARM_MOV_SI(rd, rd, SRTYPE_ASL, val), ctx);
762 } else {
763 if (val == 32)
764 emit(ARM_MOV_R(rm, rd), ctx);
765 else
766 emit(ARM_MOV_SI(rm, rd, SRTYPE_ASL, val - 32), ctx);
767 emit(ARM_EOR_R(rd, rd, rd), ctx);
768 }
769
770 if (dstk) {
771 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
772 emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
773 }
774}
775
776/* dst = dst >> val */
777static inline void emit_a32_lsr_i64(const u8 dst[], bool dstk,
778 const u32 val, struct jit_ctx *ctx) {
779 const u8 *tmp = bpf2a32[TMP_REG_1];
780 const u8 *tmp2 = bpf2a32[TMP_REG_2];
781 /* Setup operands */
782 u8 rd = dstk ? tmp[1] : dst_lo;
783 u8 rm = dstk ? tmp[0] : dst_hi;
784
785 if (dstk) {
786 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
787 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
788 }
789
790 /* Do LSR operation */
791 if (val < 32) {
792 emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
793 emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
794 emit(ARM_MOV_SI(rm, rm, SRTYPE_LSR, val), ctx);
795 } else if (val == 32) {
796 emit(ARM_MOV_R(rd, rm), ctx);
797 emit(ARM_MOV_I(rm, 0), ctx);
798 } else {
799 emit(ARM_MOV_SI(rd, rm, SRTYPE_LSR, val - 32), ctx);
800 emit(ARM_MOV_I(rm, 0), ctx);
801 }
802
803 if (dstk) {
804 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
805 emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
806 }
807}
808
809/* dst = dst >> val (signed) */
810static inline void emit_a32_arsh_i64(const u8 dst[], bool dstk,
811 const u32 val, struct jit_ctx *ctx){
812 const u8 *tmp = bpf2a32[TMP_REG_1];
813 const u8 *tmp2 = bpf2a32[TMP_REG_2];
814 /* Setup operands */
815 u8 rd = dstk ? tmp[1] : dst_lo;
816 u8 rm = dstk ? tmp[0] : dst_hi;
817
818 if (dstk) {
819 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
820 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
821 }
822
823 /* Do ARSH operation */
824 if (val < 32) {
825 emit(ARM_MOV_SI(tmp2[1], rd, SRTYPE_LSR, val), ctx);
826 emit(ARM_ORR_SI(rd, tmp2[1], rm, SRTYPE_ASL, 32 - val), ctx);
827 emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, val), ctx);
828 } else if (val == 32) {
829 emit(ARM_MOV_R(rd, rm), ctx);
830 emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
831 } else {
832 emit(ARM_MOV_SI(rd, rm, SRTYPE_ASR, val - 32), ctx);
833 emit(ARM_MOV_SI(rm, rm, SRTYPE_ASR, 31), ctx);
834 }
835
836 if (dstk) {
837 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
838 emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
839 }
840}
841
842static inline void emit_a32_mul_r64(const u8 dst[], const u8 src[], bool dstk,
843 bool sstk, struct jit_ctx *ctx) {
844 const u8 *tmp = bpf2a32[TMP_REG_1];
845 const u8 *tmp2 = bpf2a32[TMP_REG_2];
846 /* Setup operands for multiplication */
847 u8 rd = dstk ? tmp[1] : dst_lo;
848 u8 rm = dstk ? tmp[0] : dst_hi;
849 u8 rt = sstk ? tmp2[1] : src_lo;
850 u8 rn = sstk ? tmp2[0] : src_hi;
851
852 if (dstk) {
853 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
854 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
855 }
856 if (sstk) {
857 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)), ctx);
858 emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_hi)), ctx);
859 }
860
861 /* Do Multiplication */
862 emit(ARM_MUL(ARM_IP, rd, rn), ctx);
863 emit(ARM_MUL(ARM_LR, rm, rt), ctx);
864 emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);
865
866 emit(ARM_UMULL(ARM_IP, rm, rd, rt), ctx);
867 emit(ARM_ADD_R(rm, ARM_LR, rm), ctx);
868 if (dstk) {
869 emit(ARM_STR_I(ARM_IP, ARM_SP, STACK_VAR(dst_lo)), ctx);
870 emit(ARM_STR_I(rm, ARM_SP, STACK_VAR(dst_hi)), ctx);
871 } else {
872 emit(ARM_MOV_R(rd, ARM_IP), ctx);
873 }
874}
875
876/* *(size *)(dst + off) = src */
877static inline void emit_str_r(const u8 dst, const u8 src, bool dstk,
878 const s32 off, struct jit_ctx *ctx, const u8 sz){
879 const u8 *tmp = bpf2a32[TMP_REG_1];
880 u8 rd = dstk ? tmp[1] : dst;
881
882 if (dstk)
883 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst)), ctx);
884 if (off) {
885 emit_a32_mov_i(tmp[0], off, false, ctx);
886 emit(ARM_ADD_R(tmp[0], rd, tmp[0]), ctx);
887 rd = tmp[0];
888 }
889 switch (sz) {
890 case BPF_W:
891 /* Store a Word */
892 emit(ARM_STR_I(src, rd, 0), ctx);
893 break;
894 case BPF_H:
895 /* Store a HalfWord */
896 emit(ARM_STRH_I(src, rd, 0), ctx);
897 break;
898 case BPF_B:
899 /* Store a Byte */
900 emit(ARM_STRB_I(src, rd, 0), ctx);
901 break;
902 }
903}
904
905/* dst = *(size*)(src + off) */
906static inline void emit_ldx_r(const u8 dst[], const u8 src, bool dstk,
907 s32 off, struct jit_ctx *ctx, const u8 sz){
908 const u8 *tmp = bpf2a32[TMP_REG_1];
909 const u8 *rd = dstk ? tmp : dst;
910 u8 rm = src;
911 s32 off_max;
912
913 if (sz == BPF_H)
914 off_max = 0xff;
915 else
916 off_max = 0xfff;
917
918 if (off < 0 || off > off_max) {
919 emit_a32_mov_i(tmp[0], off, false, ctx);
920 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
921 rm = tmp[0];
922 off = 0;
923 } else if (rd[1] == rm) {
924 emit(ARM_MOV_R(tmp[0], rm), ctx);
925 rm = tmp[0];
926 }
927 switch (sz) {
928 case BPF_B:
929 /* Load a Byte */
930 emit(ARM_LDRB_I(rd[1], rm, off), ctx);
931 emit_a32_mov_i(dst[0], 0, dstk, ctx);
932 break;
933 case BPF_H:
934 /* Load a HalfWord */
935 emit(ARM_LDRH_I(rd[1], rm, off), ctx);
936 emit_a32_mov_i(dst[0], 0, dstk, ctx);
937 break;
938 case BPF_W:
939 /* Load a Word */
940 emit(ARM_LDR_I(rd[1], rm, off), ctx);
941 emit_a32_mov_i(dst[0], 0, dstk, ctx);
942 break;
943 case BPF_DW:
944 /* Load a Double Word */
945 emit(ARM_LDR_I(rd[1], rm, off), ctx);
946 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
947 break;
948 }
949 if (dstk)
950 emit(ARM_STR_I(rd[1], ARM_SP, STACK_VAR(dst[1])), ctx);
951 if (dstk && sz == BPF_DW)
952 emit(ARM_STR_I(rd[0], ARM_SP, STACK_VAR(dst[0])), ctx);
953}
954
955/* Arithmatic Operation */
956static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm,
957 const u8 rn, struct jit_ctx *ctx, u8 op) {
958 switch (op) {
959 case BPF_JSET:
960 emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
961 emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
962 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
963 break;
964 case BPF_JEQ:
965 case BPF_JNE:
966 case BPF_JGT:
967 case BPF_JGE:
968 case BPF_JLE:
969 case BPF_JLT:
970 emit(ARM_CMP_R(rd, rm), ctx);
971 _emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
972 break;
973 case BPF_JSLE:
974 case BPF_JSGT:
975 emit(ARM_CMP_R(rn, rt), ctx);
976 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
977 break;
978 case BPF_JSLT:
979 case BPF_JSGE:
980 emit(ARM_CMP_R(rt, rn), ctx);
981 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
982 break;
983 }
984}
985
986static int out_offset = -1; /* initialized on the first pass of build_body() */
987static int emit_bpf_tail_call(struct jit_ctx *ctx)
988{
989
990 /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */
991 const u8 *r2 = bpf2a32[BPF_REG_2];
992 const u8 *r3 = bpf2a32[BPF_REG_3];
993 const u8 *tmp = bpf2a32[TMP_REG_1];
994 const u8 *tmp2 = bpf2a32[TMP_REG_2];
995 const u8 *tcc = bpf2a32[TCALL_CNT];
996 const int idx0 = ctx->idx;
997#define cur_offset (ctx->idx - idx0)
998#define jmp_offset (out_offset - (cur_offset) - 2)
999 u32 off, lo, hi;
1000
1001 /* if (index >= array->map.max_entries)
1002 * goto out;
1003 */
1004 off = offsetof(struct bpf_array, map.max_entries);
1005 /* array->map.max_entries */
1006 emit_a32_mov_i(tmp[1], off, false, ctx);
1007 emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
1008 emit(ARM_LDR_R(tmp[1], tmp2[1], tmp[1]), ctx);
1009 /* index is 32-bit for arrays */
1010 emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
1011 /* index >= array->map.max_entries */
1012 emit(ARM_CMP_R(tmp2[1], tmp[1]), ctx);
1013 _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1014
1015 /* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
1016 * goto out;
1017 * tail_call_cnt++;
1018 */
1019 lo = (u32)MAX_TAIL_CALL_CNT;
1020 hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32);
1021 emit(ARM_LDR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
1022 emit(ARM_LDR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
1023 emit(ARM_CMP_I(tmp[0], hi), ctx);
1024 _emit(ARM_COND_EQ, ARM_CMP_I(tmp[1], lo), ctx);
1025 _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1026 emit(ARM_ADDS_I(tmp[1], tmp[1], 1), ctx);
1027 emit(ARM_ADC_I(tmp[0], tmp[0], 0), ctx);
1028 emit(ARM_STR_I(tmp[1], ARM_SP, STACK_VAR(tcc[1])), ctx);
1029 emit(ARM_STR_I(tmp[0], ARM_SP, STACK_VAR(tcc[0])), ctx);
1030
1031 /* prog = array->ptrs[index]
1032 * if (prog == NULL)
1033 * goto out;
1034 */
1035 off = offsetof(struct bpf_array, ptrs);
1036 emit_a32_mov_i(tmp[1], off, false, ctx);
1037 emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r2[1])), ctx);
1038 emit(ARM_ADD_R(tmp[1], tmp2[1], tmp[1]), ctx);
1039 emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(r3[1])), ctx);
1040 emit(ARM_MOV_SI(tmp[0], tmp2[1], SRTYPE_ASL, 2), ctx);
1041 emit(ARM_LDR_R(tmp[1], tmp[1], tmp[0]), ctx);
1042 emit(ARM_CMP_I(tmp[1], 0), ctx);
1043 _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1044
1045 /* goto *(prog->bpf_func + prologue_size); */
1046 off = offsetof(struct bpf_prog, bpf_func);
1047 emit_a32_mov_i(tmp2[1], off, false, ctx);
1048 emit(ARM_LDR_R(tmp[1], tmp[1], tmp2[1]), ctx);
1049 emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
1050 emit_bx_r(tmp[1], ctx);
1051
1052 /* out: */
1053 if (out_offset == -1)
1054 out_offset = cur_offset;
1055 if (cur_offset != out_offset) {
1056 pr_err_once("tail_call out_offset = %d, expected %d!\n",
1057 cur_offset, out_offset);
1058 return -1;
1059 }
1060 return 0;
1061#undef cur_offset
1062#undef jmp_offset
1063}
1064
1065/* 0xabcd => 0xcdab */
1066static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1067{
1068#if __LINUX_ARM_ARCH__ < 6
1069 const u8 *tmp2 = bpf2a32[TMP_REG_2];
1070
1071 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1072 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);
1073 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1074 emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);
1075#else /* ARMv6+ */
1076 emit(ARM_REV16(rd, rn), ctx);
1077#endif
1078}
1079
1080/* 0xabcdefgh => 0xghefcdab */
1081static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1082{
1083#if __LINUX_ARM_ARCH__ < 6
1084 const u8 *tmp2 = bpf2a32[TMP_REG_2];
1085
1086 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1087 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);
1088 emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);
1089
1090 emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);
1091 emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);
1092 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);
1093 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1094 emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);
1095 emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);
1096 emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);
1097
1098#else /* ARMv6+ */
1099 emit(ARM_REV(rd, rn), ctx);
1100#endif
1101}
1102
1103// push the scratch stack register on top of the stack
1104static inline void emit_push_r64(const u8 src[], const u8 shift,
1105 struct jit_ctx *ctx)
1106{
1107 const u8 *tmp2 = bpf2a32[TMP_REG_2];
1108 u16 reg_set = 0;
1109
1110 emit(ARM_LDR_I(tmp2[1], ARM_SP, STACK_VAR(src[1]+shift)), ctx);
1111 emit(ARM_LDR_I(tmp2[0], ARM_SP, STACK_VAR(src[0]+shift)), ctx);
1112
1113 reg_set = (1 << tmp2[1]) | (1 << tmp2[0]);
1114 emit(ARM_PUSH(reg_set), ctx);
1115}
1116
1117static void build_prologue(struct jit_ctx *ctx)
1118{
1119 const u8 r0 = bpf2a32[BPF_REG_0][1];
1120 const u8 r2 = bpf2a32[BPF_REG_1][1];
1121 const u8 r3 = bpf2a32[BPF_REG_1][0];
1122 const u8 r4 = bpf2a32[BPF_REG_6][1];
1123 const u8 fplo = bpf2a32[BPF_REG_FP][1];
1124 const u8 fphi = bpf2a32[BPF_REG_FP][0];
1125 const u8 *tcc = bpf2a32[TCALL_CNT];
1126
1127 /* Save callee saved registers. */
1128#ifdef CONFIG_FRAME_POINTER
1129 u16 reg_set = CALLEE_PUSH_MASK | 1 << ARM_IP | 1 << ARM_PC;
1130 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);
1131 emit(ARM_PUSH(reg_set), ctx);
1132 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);
1133#else
1134 emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);
1135 emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);
1136#endif
1137 /* Save frame pointer for later */
1138 emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx);
1139
1140 ctx->stack_size = imm8m(STACK_SIZE);
1141
1142 /* Set up function call stack */
1143 emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
1144
1145 /* Set up BPF prog stack base register */
1146 emit_a32_mov_r(fplo, ARM_IP, true, false, ctx);
1147 emit_a32_mov_i(fphi, 0, true, ctx);
1148
1149 /* mov r4, 0 */
1150 emit(ARM_MOV_I(r4, 0), ctx);
1151
1152 /* Move BPF_CTX to BPF_R1 */
1153 emit(ARM_MOV_R(r3, r4), ctx);
1154 emit(ARM_MOV_R(r2, r0), ctx);
1155 /* Initialize Tail Count */
1156 emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[0])), ctx);
1157 emit(ARM_STR_I(r4, ARM_SP, STACK_VAR(tcc[1])), ctx);
1158 /* end of prologue */
1159}
1160
1161/* restore callee saved registers. */
1162static void build_epilogue(struct jit_ctx *ctx)
1163{
1164#ifdef CONFIG_FRAME_POINTER
1165 /* When using frame pointers, some additional registers need to
1166 * be loaded. */
1167 u16 reg_set = CALLEE_POP_MASK | 1 << ARM_SP;
1168 emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);
1169 emit(ARM_LDM(ARM_SP, reg_set), ctx);
1170#else
1171 /* Restore callee saved registers. */
1172 emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);
1173 emit(ARM_POP(CALLEE_POP_MASK), ctx);
1174#endif
1175}
1176
1177/*
1178 * Convert an eBPF instruction to native instruction, i.e
1179 * JITs an eBPF instruction.
1180 * Returns :
1181 * 0 - Successfully JITed an 8-byte eBPF instruction
1182 * >0 - Successfully JITed a 16-byte eBPF instruction
1183 * <0 - Failed to JIT.
1184 */
1185static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
1186{
1187 const u8 code = insn->code;
1188 const u8 *dst = bpf2a32[insn->dst_reg];
1189 const u8 *src = bpf2a32[insn->src_reg];
1190 const u8 *tmp = bpf2a32[TMP_REG_1];
1191 const u8 *tmp2 = bpf2a32[TMP_REG_2];
1192 const s16 off = insn->off;
1193 const s32 imm = insn->imm;
1194 const int i = insn - ctx->prog->insnsi;
1195 const bool is64 = BPF_CLASS(code) == BPF_ALU64;
1196 const bool dstk = is_on_stack(insn->dst_reg);
1197 const bool sstk = is_on_stack(insn->src_reg);
1198 u8 rd, rt, rm, rn;
1199 s32 jmp_offset;
1200
1201#define check_imm(bits, imm) do { \
1202 if ((((imm) > 0) && ((imm) >> (bits))) || \
1203 (((imm) < 0) && (~(imm) >> (bits)))) { \
1204 pr_info("[%2d] imm=%d(0x%x) out of range\n", \
1205 i, imm, imm); \
1206 return -EINVAL; \
1207 } \
1208} while (0)
1209#define check_imm24(imm) check_imm(24, imm)
1210
1211 switch (code) {
1212 /* ALU operations */
1213
1214 /* dst = src */
1215 case BPF_ALU | BPF_MOV | BPF_K:
1216 case BPF_ALU | BPF_MOV | BPF_X:
1217 case BPF_ALU64 | BPF_MOV | BPF_K:
1218 case BPF_ALU64 | BPF_MOV | BPF_X:
1219 switch (BPF_SRC(code)) {
1220 case BPF_X:
1221 emit_a32_mov_r64(is64, dst, src, dstk, sstk, ctx);
1222 break;
1223 case BPF_K:
1224 /* Sign-extend immediate value to destination reg */
1225 emit_a32_mov_i64(is64, dst, imm, dstk, ctx);
1226 break;
1227 }
1228 break;
1229 /* dst = dst + src/imm */
1230 /* dst = dst - src/imm */
1231 /* dst = dst | src/imm */
1232 /* dst = dst & src/imm */
1233 /* dst = dst ^ src/imm */
1234 /* dst = dst * src/imm */
1235 /* dst = dst << src */
1236 /* dst = dst >> src */
1237 case BPF_ALU | BPF_ADD | BPF_K:
1238 case BPF_ALU | BPF_ADD | BPF_X:
1239 case BPF_ALU | BPF_SUB | BPF_K:
1240 case BPF_ALU | BPF_SUB | BPF_X:
1241 case BPF_ALU | BPF_OR | BPF_K:
1242 case BPF_ALU | BPF_OR | BPF_X:
1243 case BPF_ALU | BPF_AND | BPF_K:
1244 case BPF_ALU | BPF_AND | BPF_X:
1245 case BPF_ALU | BPF_XOR | BPF_K:
1246 case BPF_ALU | BPF_XOR | BPF_X:
1247 case BPF_ALU | BPF_MUL | BPF_K:
1248 case BPF_ALU | BPF_MUL | BPF_X:
1249 case BPF_ALU | BPF_LSH | BPF_X:
1250 case BPF_ALU | BPF_RSH | BPF_X:
1251 case BPF_ALU | BPF_ARSH | BPF_K:
1252 case BPF_ALU | BPF_ARSH | BPF_X:
1253 case BPF_ALU64 | BPF_ADD | BPF_K:
1254 case BPF_ALU64 | BPF_ADD | BPF_X:
1255 case BPF_ALU64 | BPF_SUB | BPF_K:
1256 case BPF_ALU64 | BPF_SUB | BPF_X:
1257 case BPF_ALU64 | BPF_OR | BPF_K:
1258 case BPF_ALU64 | BPF_OR | BPF_X:
1259 case BPF_ALU64 | BPF_AND | BPF_K:
1260 case BPF_ALU64 | BPF_AND | BPF_X:
1261 case BPF_ALU64 | BPF_XOR | BPF_K:
1262 case BPF_ALU64 | BPF_XOR | BPF_X:
1263 switch (BPF_SRC(code)) {
1264 case BPF_X:
1265 emit_a32_alu_r64(is64, dst, src, dstk, sstk,
1266 ctx, BPF_OP(code));
1267 break;
1268 case BPF_K:
1269 /* Move immediate value to the temporary register
1270 * and then do the ALU operation on the temporary
1271 * register as this will sign-extend the immediate
1272 * value into temporary reg and then it would be
1273 * safe to do the operation on it.
1274 */
1275 emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
1276 emit_a32_alu_r64(is64, dst, tmp2, dstk, false,
1277 ctx, BPF_OP(code));
1278 break;
1279 }
1280 break;
1281 /* dst = dst / src(imm) */
1282 /* dst = dst % src(imm) */
1283 case BPF_ALU | BPF_DIV | BPF_K:
1284 case BPF_ALU | BPF_DIV | BPF_X:
1285 case BPF_ALU | BPF_MOD | BPF_K:
1286 case BPF_ALU | BPF_MOD | BPF_X:
1287 rt = src_lo;
1288 rd = dstk ? tmp2[1] : dst_lo;
1289 if (dstk)
1290 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
1291 switch (BPF_SRC(code)) {
1292 case BPF_X:
1293 rt = sstk ? tmp2[0] : rt;
1294 if (sstk)
1295 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(src_lo)),
1296 ctx);
1297 break;
1298 case BPF_K:
1299 rt = tmp2[0];
1300 emit_a32_mov_i(rt, imm, false, ctx);
1301 break;
1302 }
1303 emit_udivmod(rd, rd, rt, ctx, BPF_OP(code));
1304 if (dstk)
1305 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_lo)), ctx);
1306 emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1307 break;
1308 case BPF_ALU64 | BPF_DIV | BPF_K:
1309 case BPF_ALU64 | BPF_DIV | BPF_X:
1310 case BPF_ALU64 | BPF_MOD | BPF_K:
1311 case BPF_ALU64 | BPF_MOD | BPF_X:
1312 goto notyet;
1313 /* dst = dst >> imm */
1314 /* dst = dst << imm */
1315 case BPF_ALU | BPF_RSH | BPF_K:
1316 case BPF_ALU | BPF_LSH | BPF_K:
1317 if (unlikely(imm > 31))
1318 return -EINVAL;
1319 if (imm)
1320 emit_a32_alu_i(dst_lo, imm, dstk, ctx, BPF_OP(code));
1321 emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1322 break;
1323 /* dst = dst << imm */
1324 case BPF_ALU64 | BPF_LSH | BPF_K:
1325 if (unlikely(imm > 63))
1326 return -EINVAL;
1327 emit_a32_lsh_i64(dst, dstk, imm, ctx);
1328 break;
1329 /* dst = dst >> imm */
1330 case BPF_ALU64 | BPF_RSH | BPF_K:
1331 if (unlikely(imm > 63))
1332 return -EINVAL;
1333 emit_a32_lsr_i64(dst, dstk, imm, ctx);
1334 break;
1335 /* dst = dst << src */
1336 case BPF_ALU64 | BPF_LSH | BPF_X:
1337 emit_a32_lsh_r64(dst, src, dstk, sstk, ctx);
1338 break;
1339 /* dst = dst >> src */
1340 case BPF_ALU64 | BPF_RSH | BPF_X:
1341 emit_a32_lsr_r64(dst, src, dstk, sstk, ctx);
1342 break;
1343 /* dst = dst >> src (signed) */
1344 case BPF_ALU64 | BPF_ARSH | BPF_X:
1345 emit_a32_arsh_r64(dst, src, dstk, sstk, ctx);
1346 break;
1347 /* dst = dst >> imm (signed) */
1348 case BPF_ALU64 | BPF_ARSH | BPF_K:
1349 if (unlikely(imm > 63))
1350 return -EINVAL;
1351 emit_a32_arsh_i64(dst, dstk, imm, ctx);
1352 break;
1353 /* dst = ~dst */
1354 case BPF_ALU | BPF_NEG:
1355 emit_a32_alu_i(dst_lo, 0, dstk, ctx, BPF_OP(code));
1356 emit_a32_mov_i(dst_hi, 0, dstk, ctx);
1357 break;
1358 /* dst = ~dst (64 bit) */
1359 case BPF_ALU64 | BPF_NEG:
1360 emit_a32_neg64(dst, dstk, ctx);
1361 break;
1362 /* dst = dst * src/imm */
1363 case BPF_ALU64 | BPF_MUL | BPF_X:
1364 case BPF_ALU64 | BPF_MUL | BPF_K:
1365 switch (BPF_SRC(code)) {
1366 case BPF_X:
1367 emit_a32_mul_r64(dst, src, dstk, sstk, ctx);
1368 break;
1369 case BPF_K:
1370 /* Move immediate value to the temporary register
1371 * and then do the multiplication on it as this
1372 * will sign-extend the immediate value into temp
1373 * reg then it would be safe to do the operation
1374 * on it.
1375 */
1376 emit_a32_mov_i64(is64, tmp2, imm, false, ctx);
1377 emit_a32_mul_r64(dst, tmp2, dstk, false, ctx);
1378 break;
1379 }
1380 break;
1381 /* dst = htole(dst) */
1382 /* dst = htobe(dst) */
1383 case BPF_ALU | BPF_END | BPF_FROM_LE:
1384 case BPF_ALU | BPF_END | BPF_FROM_BE:
1385 rd = dstk ? tmp[0] : dst_hi;
1386 rt = dstk ? tmp[1] : dst_lo;
1387 if (dstk) {
1388 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1389 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1390 }
1391 if (BPF_SRC(code) == BPF_FROM_LE)
1392 goto emit_bswap_uxt;
1393 switch (imm) {
1394 case 16:
1395 emit_rev16(rt, rt, ctx);
1396 goto emit_bswap_uxt;
1397 case 32:
1398 emit_rev32(rt, rt, ctx);
1399 goto emit_bswap_uxt;
1400 case 64:
1401 emit_rev32(ARM_LR, rt, ctx);
1402 emit_rev32(rt, rd, ctx);
1403 emit(ARM_MOV_R(rd, ARM_LR), ctx);
1404 break;
1405 }
1406 goto exit;
1407emit_bswap_uxt:
1408 switch (imm) {
1409 case 16:
1410 /* zero-extend 16 bits into 64 bits */
1411#if __LINUX_ARM_ARCH__ < 6
1412 emit_a32_mov_i(tmp2[1], 0xffff, false, ctx);
1413 emit(ARM_AND_R(rt, rt, tmp2[1]), ctx);
1414#else /* ARMv6+ */
1415 emit(ARM_UXTH(rt, rt), ctx);
1416#endif
1417 emit(ARM_EOR_R(rd, rd, rd), ctx);
1418 break;
1419 case 32:
1420 /* zero-extend 32 bits into 64 bits */
1421 emit(ARM_EOR_R(rd, rd, rd), ctx);
1422 break;
1423 case 64:
1424 /* nop */
1425 break;
1426 }
1427exit:
1428 if (dstk) {
1429 emit(ARM_STR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1430 emit(ARM_STR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1431 }
1432 break;
1433 /* dst = imm64 */
1434 case BPF_LD | BPF_IMM | BPF_DW:
1435 {
1436 const struct bpf_insn insn1 = insn[1];
1437 u32 hi, lo = imm;
1438
1439 hi = insn1.imm;
1440 emit_a32_mov_i(dst_lo, lo, dstk, ctx);
1441 emit_a32_mov_i(dst_hi, hi, dstk, ctx);
1442
1443 return 1;
1444 }
1445 /* LDX: dst = *(size *)(src + off) */
1446 case BPF_LDX | BPF_MEM | BPF_W:
1447 case BPF_LDX | BPF_MEM | BPF_H:
1448 case BPF_LDX | BPF_MEM | BPF_B:
1449 case BPF_LDX | BPF_MEM | BPF_DW:
1450 rn = sstk ? tmp2[1] : src_lo;
1451 if (sstk)
1452 emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1453 emit_ldx_r(dst, rn, dstk, off, ctx, BPF_SIZE(code));
1454 break;
1455 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */
1456 case BPF_LD | BPF_ABS | BPF_W:
1457 case BPF_LD | BPF_ABS | BPF_H:
1458 case BPF_LD | BPF_ABS | BPF_B:
1459 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */
1460 case BPF_LD | BPF_IND | BPF_W:
1461 case BPF_LD | BPF_IND | BPF_H:
1462 case BPF_LD | BPF_IND | BPF_B:
1463 {
1464 const u8 r4 = bpf2a32[BPF_REG_6][1]; /* r4 = ptr to sk_buff */
1465 const u8 r0 = bpf2a32[BPF_REG_0][1]; /*r0: struct sk_buff *skb*/
1466 /* rtn value */
1467 const u8 r1 = bpf2a32[BPF_REG_0][0]; /* r1: int k */
1468 const u8 r2 = bpf2a32[BPF_REG_1][1]; /* r2: unsigned int size */
1469 const u8 r3 = bpf2a32[BPF_REG_1][0]; /* r3: void *buffer */
1470 const u8 r6 = bpf2a32[TMP_REG_1][1]; /* r6: void *(*func)(..) */
1471 int size;
1472
1473 /* Setting up first argument */
1474 emit(ARM_MOV_R(r0, r4), ctx);
1475
1476 /* Setting up second argument */
1477 emit_a32_mov_i(r1, imm, false, ctx);
1478 if (BPF_MODE(code) == BPF_IND)
1479 emit_a32_alu_r(r1, src_lo, false, sstk, ctx,
1480 false, false, BPF_ADD);
1481
1482 /* Setting up third argument */
1483 switch (BPF_SIZE(code)) {
1484 case BPF_W:
1485 size = 4;
1486 break;
1487 case BPF_H:
1488 size = 2;
1489 break;
1490 case BPF_B:
1491 size = 1;
1492 break;
1493 default:
1494 return -EINVAL;
1495 }
1496 emit_a32_mov_i(r2, size, false, ctx);
1497
1498 /* Setting up fourth argument */
1499 emit(ARM_ADD_I(r3, ARM_SP, imm8m(SKB_BUFFER)), ctx);
1500
1501 /* Setting up function pointer to call */
1502 emit_a32_mov_i(r6, (unsigned int)bpf_load_pointer, false, ctx);
1503 emit_blx_r(r6, ctx);
1504
1505 emit(ARM_EOR_R(r1, r1, r1), ctx);
1506 /* Check if return address is NULL or not.
1507 * if NULL then jump to epilogue
1508 * else continue to load the value from retn address
1509 */
1510 emit(ARM_CMP_I(r0, 0), ctx);
1511 jmp_offset = epilogue_offset(ctx);
1512 check_imm24(jmp_offset);
1513 _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1514
1515 /* Load value from the address */
1516 switch (BPF_SIZE(code)) {
1517 case BPF_W:
1518 emit(ARM_LDR_I(r0, r0, 0), ctx);
1519 emit_rev32(r0, r0, ctx);
1520 break;
1521 case BPF_H:
1522 emit(ARM_LDRH_I(r0, r0, 0), ctx);
1523 emit_rev16(r0, r0, ctx);
1524 break;
1525 case BPF_B:
1526 emit(ARM_LDRB_I(r0, r0, 0), ctx);
1527 /* No need to reverse */
1528 break;
1529 }
1530 break;
1531 }
1532 /* ST: *(size *)(dst + off) = imm */
1533 case BPF_ST | BPF_MEM | BPF_W:
1534 case BPF_ST | BPF_MEM | BPF_H:
1535 case BPF_ST | BPF_MEM | BPF_B:
1536 case BPF_ST | BPF_MEM | BPF_DW:
1537 switch (BPF_SIZE(code)) {
1538 case BPF_DW:
1539 /* Sign-extend immediate value into temp reg */
1540 emit_a32_mov_i64(true, tmp2, imm, false, ctx);
1541 emit_str_r(dst_lo, tmp2[1], dstk, off, ctx, BPF_W);
1542 emit_str_r(dst_lo, tmp2[0], dstk, off+4, ctx, BPF_W);
1543 break;
1544 case BPF_W:
1545 case BPF_H:
1546 case BPF_B:
1547 emit_a32_mov_i(tmp2[1], imm, false, ctx);
1548 emit_str_r(dst_lo, tmp2[1], dstk, off, ctx,
1549 BPF_SIZE(code));
1550 break;
1551 }
1552 break;
1553 /* STX XADD: lock *(u32 *)(dst + off) += src */
1554 case BPF_STX | BPF_XADD | BPF_W:
1555 /* STX XADD: lock *(u64 *)(dst + off) += src */
1556 case BPF_STX | BPF_XADD | BPF_DW:
1557 goto notyet;
1558 /* STX: *(size *)(dst + off) = src */
1559 case BPF_STX | BPF_MEM | BPF_W:
1560 case BPF_STX | BPF_MEM | BPF_H:
1561 case BPF_STX | BPF_MEM | BPF_B:
1562 case BPF_STX | BPF_MEM | BPF_DW:
1563 {
1564 u8 sz = BPF_SIZE(code);
1565
1566 rn = sstk ? tmp2[1] : src_lo;
1567 rm = sstk ? tmp2[0] : src_hi;
1568 if (sstk) {
1569 emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1570 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
1571 }
1572
1573 /* Store the value */
1574 if (BPF_SIZE(code) == BPF_DW) {
1575 emit_str_r(dst_lo, rn, dstk, off, ctx, BPF_W);
1576 emit_str_r(dst_lo, rm, dstk, off+4, ctx, BPF_W);
1577 } else {
1578 emit_str_r(dst_lo, rn, dstk, off, ctx, sz);
1579 }
1580 break;
1581 }
1582 /* PC += off if dst == src */
1583 /* PC += off if dst > src */
1584 /* PC += off if dst >= src */
1585 /* PC += off if dst < src */
1586 /* PC += off if dst <= src */
1587 /* PC += off if dst != src */
1588 /* PC += off if dst > src (signed) */
1589 /* PC += off if dst >= src (signed) */
1590 /* PC += off if dst < src (signed) */
1591 /* PC += off if dst <= src (signed) */
1592 /* PC += off if dst & src */
1593 case BPF_JMP | BPF_JEQ | BPF_X:
1594 case BPF_JMP | BPF_JGT | BPF_X:
1595 case BPF_JMP | BPF_JGE | BPF_X:
1596 case BPF_JMP | BPF_JNE | BPF_X:
1597 case BPF_JMP | BPF_JSGT | BPF_X:
1598 case BPF_JMP | BPF_JSGE | BPF_X:
1599 case BPF_JMP | BPF_JSET | BPF_X:
1600 case BPF_JMP | BPF_JLE | BPF_X:
1601 case BPF_JMP | BPF_JLT | BPF_X:
1602 case BPF_JMP | BPF_JSLT | BPF_X:
1603 case BPF_JMP | BPF_JSLE | BPF_X:
1604 /* Setup source registers */
1605 rm = sstk ? tmp2[0] : src_hi;
1606 rn = sstk ? tmp2[1] : src_lo;
1607 if (sstk) {
1608 emit(ARM_LDR_I(rn, ARM_SP, STACK_VAR(src_lo)), ctx);
1609 emit(ARM_LDR_I(rm, ARM_SP, STACK_VAR(src_hi)), ctx);
1610 }
1611 goto go_jmp;
1612 /* PC += off if dst == imm */
1613 /* PC += off if dst > imm */
1614 /* PC += off if dst >= imm */
1615 /* PC += off if dst < imm */
1616 /* PC += off if dst <= imm */
1617 /* PC += off if dst != imm */
1618 /* PC += off if dst > imm (signed) */
1619 /* PC += off if dst >= imm (signed) */
1620 /* PC += off if dst < imm (signed) */
1621 /* PC += off if dst <= imm (signed) */
1622 /* PC += off if dst & imm */
1623 case BPF_JMP | BPF_JEQ | BPF_K:
1624 case BPF_JMP | BPF_JGT | BPF_K:
1625 case BPF_JMP | BPF_JGE | BPF_K:
1626 case BPF_JMP | BPF_JNE | BPF_K:
1627 case BPF_JMP | BPF_JSGT | BPF_K:
1628 case BPF_JMP | BPF_JSGE | BPF_K:
1629 case BPF_JMP | BPF_JSET | BPF_K:
1630 case BPF_JMP | BPF_JLT | BPF_K:
1631 case BPF_JMP | BPF_JLE | BPF_K:
1632 case BPF_JMP | BPF_JSLT | BPF_K:
1633 case BPF_JMP | BPF_JSLE | BPF_K:
1634 if (off == 0)
1635 break;
1636 rm = tmp2[0];
1637 rn = tmp2[1];
1638 /* Sign-extend immediate value */
1639 emit_a32_mov_i64(true, tmp2, imm, false, ctx);
1640go_jmp:
1641 /* Setup destination register */
1642 rd = dstk ? tmp[0] : dst_hi;
1643 rt = dstk ? tmp[1] : dst_lo;
1644 if (dstk) {
1645 emit(ARM_LDR_I(rt, ARM_SP, STACK_VAR(dst_lo)), ctx);
1646 emit(ARM_LDR_I(rd, ARM_SP, STACK_VAR(dst_hi)), ctx);
1647 }
1648
1649 /* Check for the condition */
1650 emit_ar_r(rd, rt, rm, rn, ctx, BPF_OP(code));
1651
1652 /* Setup JUMP instruction */
1653 jmp_offset = bpf2a32_offset(i+off, i, ctx);
1654 switch (BPF_OP(code)) {
1655 case BPF_JNE:
1656 case BPF_JSET:
1657 _emit(ARM_COND_NE, ARM_B(jmp_offset), ctx);
1658 break;
1659 case BPF_JEQ:
1660 _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1661 break;
1662 case BPF_JGT:
1663 _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1664 break;
1665 case BPF_JGE:
1666 _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1667 break;
1668 case BPF_JSGT:
1669 _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1670 break;
1671 case BPF_JSGE:
1672 _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1673 break;
1674 case BPF_JLE:
1675 _emit(ARM_COND_LS, ARM_B(jmp_offset), ctx);
1676 break;
1677 case BPF_JLT:
1678 _emit(ARM_COND_CC, ARM_B(jmp_offset), ctx);
1679 break;
1680 case BPF_JSLT:
1681 _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1682 break;
1683 case BPF_JSLE:
1684 _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1685 break;
1686 }
1687 break;
1688 /* JMP OFF */
1689 case BPF_JMP | BPF_JA:
1690 {
1691 if (off == 0)
1692 break;
1693 jmp_offset = bpf2a32_offset(i+off, i, ctx);
1694 check_imm24(jmp_offset);
1695 emit(ARM_B(jmp_offset), ctx);
1696 break;
1697 }
1698 /* tail call */
1699 case BPF_JMP | BPF_TAIL_CALL:
1700 if (emit_bpf_tail_call(ctx))
1701 return -EFAULT;
1702 break;
1703 /* function call */
1704 case BPF_JMP | BPF_CALL:
1705 {
1706 const u8 *r0 = bpf2a32[BPF_REG_0];
1707 const u8 *r1 = bpf2a32[BPF_REG_1];
1708 const u8 *r2 = bpf2a32[BPF_REG_2];
1709 const u8 *r3 = bpf2a32[BPF_REG_3];
1710 const u8 *r4 = bpf2a32[BPF_REG_4];
1711 const u8 *r5 = bpf2a32[BPF_REG_5];
1712 const u32 func = (u32)__bpf_call_base + (u32)imm;
1713
1714 emit_a32_mov_r64(true, r0, r1, false, false, ctx);
1715 emit_a32_mov_r64(true, r1, r2, false, true, ctx);
1716 emit_push_r64(r5, 0, ctx);
1717 emit_push_r64(r4, 8, ctx);
1718 emit_push_r64(r3, 16, ctx);
1719
1720 emit_a32_mov_i(tmp[1], func, false, ctx);
1721 emit_blx_r(tmp[1], ctx);
1722
1723 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
1724 break;
1725 }
1726 /* function return */
1727 case BPF_JMP | BPF_EXIT:
1728 /* Optimization: when last instruction is EXIT
1729 * simply fallthrough to epilogue.
1730 */
1731 if (i == ctx->prog->len - 1)
1732 break;
1733 jmp_offset = epilogue_offset(ctx);
1734 check_imm24(jmp_offset);
1735 emit(ARM_B(jmp_offset), ctx);
1736 break;
1737notyet:
1738 pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1739 return -EFAULT;
1740 default:
1741 pr_err_once("unknown opcode %02x\n", code);
1742 return -EINVAL;
1743 }
1744
1745 if (ctx->flags & FLAG_IMM_OVERFLOW)
1746 /*
1747 * this instruction generated an overflow when
1748 * trying to access the literal pool, so
1749 * delegate this filter to the kernel interpreter.
1750 */
1751 return -1;
1752 return 0;
1753}
1754
1755static int build_body(struct jit_ctx *ctx)
1756{
1757 const struct bpf_prog *prog = ctx->prog;
1758 unsigned int i;
1759
1760 for (i = 0; i < prog->len; i++) {
1761 const struct bpf_insn *insn = &(prog->insnsi[i]);
1762 int ret;
1763
1764 ret = build_insn(insn, ctx);
1765
1766 /* It's used with loading the 64 bit immediate value. */
1767 if (ret > 0) {
1768 i++;
1769 if (ctx->target == NULL)
1770 ctx->offsets[i] = ctx->idx;
1771 continue;
1772 }
1773
1774 if (ctx->target == NULL)
1775 ctx->offsets[i] = ctx->idx;
1776
1777 /* If unsuccesfull, return with error code */
1778 if (ret)
1779 return ret;
1780 }
1781 return 0;
1782}
1783
1784static int validate_code(struct jit_ctx *ctx)
1785{
1786 int i;
1787
1788 for (i = 0; i < ctx->idx; i++) {
1789 if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF))
1790 return -1;
1791 }
1792
1793 return 0;
1794}
1795
1796void bpf_jit_compile(struct bpf_prog *prog)
1797{
1798 /* Nothing to do here. We support Internal BPF. */
1799}
1800
1801struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1802{
1803 struct bpf_prog *tmp, *orig_prog = prog;
1804 struct bpf_binary_header *header;
1805 bool tmp_blinded = false;
1806 struct jit_ctx ctx;
1807 unsigned int tmp_idx;
1808 unsigned int image_size;
1809 u8 *image_ptr;
1810
1811 /* If BPF JIT was not enabled then we must fall back to
1812 * the interpreter.
1813 */
1814 if (!prog->jit_requested)
1815 return orig_prog;
1816
1817 /* If constant blinding was enabled and we failed during blinding
1818 * then we must fall back to the interpreter. Otherwise, we save
1819 * the new JITed code.
1820 */
1821 tmp = bpf_jit_blind_constants(prog);
1822
1823 if (IS_ERR(tmp))
1824 return orig_prog;
1825 if (tmp != prog) {
1826 tmp_blinded = true;
1827 prog = tmp;
1828 }
1829
1830 memset(&ctx, 0, sizeof(ctx));
1831 ctx.prog = prog;
1832
1833 /* Not able to allocate memory for offsets[] , then
1834 * we must fall back to the interpreter
1835 */
1836 ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1837 if (ctx.offsets == NULL) {
1838 prog = orig_prog;
1839 goto out;
1840 }
1841
1842 /* 1) fake pass to find in the length of the JITed code,
1843 * to compute ctx->offsets and other context variables
1844 * needed to compute final JITed code.
1845 * Also, calculate random starting pointer/start of JITed code
1846 * which is prefixed by random number of fault instructions.
1847 *
1848 * If the first pass fails then there is no chance of it
1849 * being successful in the second pass, so just fall back
1850 * to the interpreter.
1851 */
1852 if (build_body(&ctx)) {
1853 prog = orig_prog;
1854 goto out_off;
1855 }
1856
1857 tmp_idx = ctx.idx;
1858 build_prologue(&ctx);
1859 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
1860
1861 ctx.epilogue_offset = ctx.idx;
1862
1863#if __LINUX_ARM_ARCH__ < 7
1864 tmp_idx = ctx.idx;
1865 build_epilogue(&ctx);
1866 ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4;
1867
1868 ctx.idx += ctx.imm_count;
1869 if (ctx.imm_count) {
1870 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
1871 if (ctx.imms == NULL) {
1872 prog = orig_prog;
1873 goto out_off;
1874 }
1875 }
1876#else
1877 /* there's nothing about the epilogue on ARMv7 */
1878 build_epilogue(&ctx);
1879#endif
1880 /* Now we can get the actual image size of the JITed arm code.
1881 * Currently, we are not considering the THUMB-2 instructions
1882 * for jit, although it can decrease the size of the image.
1883 *
1884 * As each arm instruction is of length 32bit, we are translating
1885 * number of JITed intructions into the size required to store these
1886 * JITed code.
1887 */
1888 image_size = sizeof(u32) * ctx.idx;
1889
1890 /* Now we know the size of the structure to make */
1891 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1892 sizeof(u32), jit_fill_hole);
1893 /* Not able to allocate memory for the structure then
1894 * we must fall back to the interpretation
1895 */
1896 if (header == NULL) {
1897 prog = orig_prog;
1898 goto out_imms;
1899 }
1900
1901 /* 2.) Actual pass to generate final JIT code */
1902 ctx.target = (u32 *) image_ptr;
1903 ctx.idx = 0;
1904
1905 build_prologue(&ctx);
1906
1907 /* If building the body of the JITed code fails somehow,
1908 * we fall back to the interpretation.
1909 */
1910 if (build_body(&ctx) < 0) {
1911 image_ptr = NULL;
1912 bpf_jit_binary_free(header);
1913 prog = orig_prog;
1914 goto out_imms;
1915 }
1916 build_epilogue(&ctx);
1917
1918 /* 3.) Extra pass to validate JITed Code */
1919 if (validate_code(&ctx)) {
1920 image_ptr = NULL;
1921 bpf_jit_binary_free(header);
1922 prog = orig_prog;
1923 goto out_imms;
1924 }
1925 flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
1926
1927 if (bpf_jit_enable > 1)
1928 /* there are 2 passes here */
1929 bpf_jit_dump(prog->len, image_size, 2, ctx.target);
1930
1931 set_memory_ro((unsigned long)header, header->pages);
1932 prog->bpf_func = (void *)ctx.target;
1933 prog->jited = 1;
1934 prog->jited_len = image_size;
1935
1936out_imms:
1937#if __LINUX_ARM_ARCH__ < 7
1938 if (ctx.imm_count)
1939 kfree(ctx.imms);
1940#endif
1941out_off:
1942 kfree(ctx.offsets);
1943out:
1944 if (tmp_blinded)
1945 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1946 tmp : orig_prog);
1947 return prog;
1948}
1949
1/*
2 * Just-In-Time compiler for BPF filters on 32bit ARM
3 *
4 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License.
9 */
10
11#include <linux/bitops.h>
12#include <linux/compiler.h>
13#include <linux/errno.h>
14#include <linux/filter.h>
15#include <linux/moduleloader.h>
16#include <linux/netdevice.h>
17#include <linux/string.h>
18#include <linux/slab.h>
19#include <linux/if_vlan.h>
20#include <asm/cacheflush.h>
21#include <asm/hwcap.h>
22#include <asm/opcodes.h>
23
24#include "bpf_jit_32.h"
25
26/*
27 * ABI:
28 *
29 * r0 scratch register
30 * r4 BPF register A
31 * r5 BPF register X
32 * r6 pointer to the skb
33 * r7 skb->data
34 * r8 skb_headlen(skb)
35 */
36
37#define r_scratch ARM_R0
38/* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */
39#define r_off ARM_R1
40#define r_A ARM_R4
41#define r_X ARM_R5
42#define r_skb ARM_R6
43#define r_skb_data ARM_R7
44#define r_skb_hl ARM_R8
45
46#define SCRATCH_SP_OFFSET 0
47#define SCRATCH_OFF(k) (SCRATCH_SP_OFFSET + 4 * (k))
48
49#define SEEN_MEM ((1 << BPF_MEMWORDS) - 1)
50#define SEEN_MEM_WORD(k) (1 << (k))
51#define SEEN_X (1 << BPF_MEMWORDS)
52#define SEEN_CALL (1 << (BPF_MEMWORDS + 1))
53#define SEEN_SKB (1 << (BPF_MEMWORDS + 2))
54#define SEEN_DATA (1 << (BPF_MEMWORDS + 3))
55
56#define FLAG_NEED_X_RESET (1 << 0)
57
58struct jit_ctx {
59 const struct sk_filter *skf;
60 unsigned idx;
61 unsigned prologue_bytes;
62 int ret0_fp_idx;
63 u32 seen;
64 u32 flags;
65 u32 *offsets;
66 u32 *target;
67#if __LINUX_ARM_ARCH__ < 7
68 u16 epilogue_bytes;
69 u16 imm_count;
70 u32 *imms;
71#endif
72};
73
74int bpf_jit_enable __read_mostly;
75
76static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset)
77{
78 u8 ret;
79 int err;
80
81 err = skb_copy_bits(skb, offset, &ret, 1);
82
83 return (u64)err << 32 | ret;
84}
85
86static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset)
87{
88 u16 ret;
89 int err;
90
91 err = skb_copy_bits(skb, offset, &ret, 2);
92
93 return (u64)err << 32 | ntohs(ret);
94}
95
96static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset)
97{
98 u32 ret;
99 int err;
100
101 err = skb_copy_bits(skb, offset, &ret, 4);
102
103 return (u64)err << 32 | ntohl(ret);
104}
105
106/*
107 * Wrapper that handles both OABI and EABI and assures Thumb2 interworking
108 * (where the assembly routines like __aeabi_uidiv could cause problems).
109 */
110static u32 jit_udiv(u32 dividend, u32 divisor)
111{
112 return dividend / divisor;
113}
114
115static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
116{
117 inst |= (cond << 28);
118 inst = __opcode_to_mem_arm(inst);
119
120 if (ctx->target != NULL)
121 ctx->target[ctx->idx] = inst;
122
123 ctx->idx++;
124}
125
126/*
127 * Emit an instruction that will be executed unconditionally.
128 */
129static inline void emit(u32 inst, struct jit_ctx *ctx)
130{
131 _emit(ARM_COND_AL, inst, ctx);
132}
133
134static u16 saved_regs(struct jit_ctx *ctx)
135{
136 u16 ret = 0;
137
138 if ((ctx->skf->len > 1) ||
139 (ctx->skf->insns[0].code == BPF_S_RET_A))
140 ret |= 1 << r_A;
141
142#ifdef CONFIG_FRAME_POINTER
143 ret |= (1 << ARM_FP) | (1 << ARM_IP) | (1 << ARM_LR) | (1 << ARM_PC);
144#else
145 if (ctx->seen & SEEN_CALL)
146 ret |= 1 << ARM_LR;
147#endif
148 if (ctx->seen & (SEEN_DATA | SEEN_SKB))
149 ret |= 1 << r_skb;
150 if (ctx->seen & SEEN_DATA)
151 ret |= (1 << r_skb_data) | (1 << r_skb_hl);
152 if (ctx->seen & SEEN_X)
153 ret |= 1 << r_X;
154
155 return ret;
156}
157
158static inline int mem_words_used(struct jit_ctx *ctx)
159{
160 /* yes, we do waste some stack space IF there are "holes" in the set" */
161 return fls(ctx->seen & SEEN_MEM);
162}
163
164static inline bool is_load_to_a(u16 inst)
165{
166 switch (inst) {
167 case BPF_S_LD_W_LEN:
168 case BPF_S_LD_W_ABS:
169 case BPF_S_LD_H_ABS:
170 case BPF_S_LD_B_ABS:
171 case BPF_S_ANC_CPU:
172 case BPF_S_ANC_IFINDEX:
173 case BPF_S_ANC_MARK:
174 case BPF_S_ANC_PROTOCOL:
175 case BPF_S_ANC_RXHASH:
176 case BPF_S_ANC_VLAN_TAG:
177 case BPF_S_ANC_VLAN_TAG_PRESENT:
178 case BPF_S_ANC_QUEUE:
179 return true;
180 default:
181 return false;
182 }
183}
184
185static void build_prologue(struct jit_ctx *ctx)
186{
187 u16 reg_set = saved_regs(ctx);
188 u16 first_inst = ctx->skf->insns[0].code;
189 u16 off;
190
191#ifdef CONFIG_FRAME_POINTER
192 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);
193 emit(ARM_PUSH(reg_set), ctx);
194 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);
195#else
196 if (reg_set)
197 emit(ARM_PUSH(reg_set), ctx);
198#endif
199
200 if (ctx->seen & (SEEN_DATA | SEEN_SKB))
201 emit(ARM_MOV_R(r_skb, ARM_R0), ctx);
202
203 if (ctx->seen & SEEN_DATA) {
204 off = offsetof(struct sk_buff, data);
205 emit(ARM_LDR_I(r_skb_data, r_skb, off), ctx);
206 /* headlen = len - data_len */
207 off = offsetof(struct sk_buff, len);
208 emit(ARM_LDR_I(r_skb_hl, r_skb, off), ctx);
209 off = offsetof(struct sk_buff, data_len);
210 emit(ARM_LDR_I(r_scratch, r_skb, off), ctx);
211 emit(ARM_SUB_R(r_skb_hl, r_skb_hl, r_scratch), ctx);
212 }
213
214 if (ctx->flags & FLAG_NEED_X_RESET)
215 emit(ARM_MOV_I(r_X, 0), ctx);
216
217 /* do not leak kernel data to userspace */
218 if ((first_inst != BPF_S_RET_K) && !(is_load_to_a(first_inst)))
219 emit(ARM_MOV_I(r_A, 0), ctx);
220
221 /* stack space for the BPF_MEM words */
222 if (ctx->seen & SEEN_MEM)
223 emit(ARM_SUB_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx);
224}
225
226static void build_epilogue(struct jit_ctx *ctx)
227{
228 u16 reg_set = saved_regs(ctx);
229
230 if (ctx->seen & SEEN_MEM)
231 emit(ARM_ADD_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx);
232
233 reg_set &= ~(1 << ARM_LR);
234
235#ifdef CONFIG_FRAME_POINTER
236 /* the first instruction of the prologue was: mov ip, sp */
237 reg_set &= ~(1 << ARM_IP);
238 reg_set |= (1 << ARM_SP);
239 emit(ARM_LDM(ARM_SP, reg_set), ctx);
240#else
241 if (reg_set) {
242 if (ctx->seen & SEEN_CALL)
243 reg_set |= 1 << ARM_PC;
244 emit(ARM_POP(reg_set), ctx);
245 }
246
247 if (!(ctx->seen & SEEN_CALL))
248 emit(ARM_BX(ARM_LR), ctx);
249#endif
250}
251
252static int16_t imm8m(u32 x)
253{
254 u32 rot;
255
256 for (rot = 0; rot < 16; rot++)
257 if ((x & ~ror32(0xff, 2 * rot)) == 0)
258 return rol32(x, 2 * rot) | (rot << 8);
259
260 return -1;
261}
262
263#if __LINUX_ARM_ARCH__ < 7
264
265static u16 imm_offset(u32 k, struct jit_ctx *ctx)
266{
267 unsigned i = 0, offset;
268 u16 imm;
269
270 /* on the "fake" run we just count them (duplicates included) */
271 if (ctx->target == NULL) {
272 ctx->imm_count++;
273 return 0;
274 }
275
276 while ((i < ctx->imm_count) && ctx->imms[i]) {
277 if (ctx->imms[i] == k)
278 break;
279 i++;
280 }
281
282 if (ctx->imms[i] == 0)
283 ctx->imms[i] = k;
284
285 /* constants go just after the epilogue */
286 offset = ctx->offsets[ctx->skf->len];
287 offset += ctx->prologue_bytes;
288 offset += ctx->epilogue_bytes;
289 offset += i * 4;
290
291 ctx->target[offset / 4] = k;
292
293 /* PC in ARM mode == address of the instruction + 8 */
294 imm = offset - (8 + ctx->idx * 4);
295
296 return imm;
297}
298
299#endif /* __LINUX_ARM_ARCH__ */
300
301/*
302 * Move an immediate that's not an imm8m to a core register.
303 */
304static inline void emit_mov_i_no8m(int rd, u32 val, struct jit_ctx *ctx)
305{
306#if __LINUX_ARM_ARCH__ < 7
307 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);
308#else
309 emit(ARM_MOVW(rd, val & 0xffff), ctx);
310 if (val > 0xffff)
311 emit(ARM_MOVT(rd, val >> 16), ctx);
312#endif
313}
314
315static inline void emit_mov_i(int rd, u32 val, struct jit_ctx *ctx)
316{
317 int imm12 = imm8m(val);
318
319 if (imm12 >= 0)
320 emit(ARM_MOV_I(rd, imm12), ctx);
321 else
322 emit_mov_i_no8m(rd, val, ctx);
323}
324
325#if __LINUX_ARM_ARCH__ < 6
326
327static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx)
328{
329 _emit(cond, ARM_LDRB_I(ARM_R3, r_addr, 1), ctx);
330 _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx);
331 _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 3), ctx);
332 _emit(cond, ARM_LSL_I(ARM_R3, ARM_R3, 16), ctx);
333 _emit(cond, ARM_LDRB_I(ARM_R0, r_addr, 2), ctx);
334 _emit(cond, ARM_ORR_S(ARM_R3, ARM_R3, ARM_R1, SRTYPE_LSL, 24), ctx);
335 _emit(cond, ARM_ORR_R(ARM_R3, ARM_R3, ARM_R2), ctx);
336 _emit(cond, ARM_ORR_S(r_res, ARM_R3, ARM_R0, SRTYPE_LSL, 8), ctx);
337}
338
339static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx)
340{
341 _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx);
342 _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 1), ctx);
343 _emit(cond, ARM_ORR_S(r_res, ARM_R2, ARM_R1, SRTYPE_LSL, 8), ctx);
344}
345
346static inline void emit_swap16(u8 r_dst, u8 r_src, struct jit_ctx *ctx)
347{
348 /* r_dst = (r_src << 8) | (r_src >> 8) */
349 emit(ARM_LSL_I(ARM_R1, r_src, 8), ctx);
350 emit(ARM_ORR_S(r_dst, ARM_R1, r_src, SRTYPE_LSR, 8), ctx);
351
352 /*
353 * we need to mask out the bits set in r_dst[23:16] due to
354 * the first shift instruction.
355 *
356 * note that 0x8ff is the encoded immediate 0x00ff0000.
357 */
358 emit(ARM_BIC_I(r_dst, r_dst, 0x8ff), ctx);
359}
360
361#else /* ARMv6+ */
362
363static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx)
364{
365 _emit(cond, ARM_LDR_I(r_res, r_addr, 0), ctx);
366#ifdef __LITTLE_ENDIAN
367 _emit(cond, ARM_REV(r_res, r_res), ctx);
368#endif
369}
370
371static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx)
372{
373 _emit(cond, ARM_LDRH_I(r_res, r_addr, 0), ctx);
374#ifdef __LITTLE_ENDIAN
375 _emit(cond, ARM_REV16(r_res, r_res), ctx);
376#endif
377}
378
379static inline void emit_swap16(u8 r_dst __maybe_unused,
380 u8 r_src __maybe_unused,
381 struct jit_ctx *ctx __maybe_unused)
382{
383#ifdef __LITTLE_ENDIAN
384 emit(ARM_REV16(r_dst, r_src), ctx);
385#endif
386}
387
388#endif /* __LINUX_ARM_ARCH__ < 6 */
389
390
391/* Compute the immediate value for a PC-relative branch. */
392static inline u32 b_imm(unsigned tgt, struct jit_ctx *ctx)
393{
394 u32 imm;
395
396 if (ctx->target == NULL)
397 return 0;
398 /*
399 * BPF allows only forward jumps and the offset of the target is
400 * still the one computed during the first pass.
401 */
402 imm = ctx->offsets[tgt] + ctx->prologue_bytes - (ctx->idx * 4 + 8);
403
404 return imm >> 2;
405}
406
407#define OP_IMM3(op, r1, r2, imm_val, ctx) \
408 do { \
409 imm12 = imm8m(imm_val); \
410 if (imm12 < 0) { \
411 emit_mov_i_no8m(r_scratch, imm_val, ctx); \
412 emit(op ## _R((r1), (r2), r_scratch), ctx); \
413 } else { \
414 emit(op ## _I((r1), (r2), imm12), ctx); \
415 } \
416 } while (0)
417
418static inline void emit_err_ret(u8 cond, struct jit_ctx *ctx)
419{
420 if (ctx->ret0_fp_idx >= 0) {
421 _emit(cond, ARM_B(b_imm(ctx->ret0_fp_idx, ctx)), ctx);
422 /* NOP to keep the size constant between passes */
423 emit(ARM_MOV_R(ARM_R0, ARM_R0), ctx);
424 } else {
425 _emit(cond, ARM_MOV_I(ARM_R0, 0), ctx);
426 _emit(cond, ARM_B(b_imm(ctx->skf->len, ctx)), ctx);
427 }
428}
429
430static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
431{
432#if __LINUX_ARM_ARCH__ < 5
433 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);
434
435 if (elf_hwcap & HWCAP_THUMB)
436 emit(ARM_BX(tgt_reg), ctx);
437 else
438 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);
439#else
440 emit(ARM_BLX_R(tgt_reg), ctx);
441#endif
442}
443
444static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx)
445{
446#if __LINUX_ARM_ARCH__ == 7
447 if (elf_hwcap & HWCAP_IDIVA) {
448 emit(ARM_UDIV(rd, rm, rn), ctx);
449 return;
450 }
451#endif
452 if (rm != ARM_R0)
453 emit(ARM_MOV_R(ARM_R0, rm), ctx);
454 if (rn != ARM_R1)
455 emit(ARM_MOV_R(ARM_R1, rn), ctx);
456
457 ctx->seen |= SEEN_CALL;
458 emit_mov_i(ARM_R3, (u32)jit_udiv, ctx);
459 emit_blx_r(ARM_R3, ctx);
460
461 if (rd != ARM_R0)
462 emit(ARM_MOV_R(rd, ARM_R0), ctx);
463}
464
465static inline void update_on_xread(struct jit_ctx *ctx)
466{
467 if (!(ctx->seen & SEEN_X))
468 ctx->flags |= FLAG_NEED_X_RESET;
469
470 ctx->seen |= SEEN_X;
471}
472
473static int build_body(struct jit_ctx *ctx)
474{
475 void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w};
476 const struct sk_filter *prog = ctx->skf;
477 const struct sock_filter *inst;
478 unsigned i, load_order, off, condt;
479 int imm12;
480 u32 k;
481
482 for (i = 0; i < prog->len; i++) {
483 inst = &(prog->insns[i]);
484 /* K as an immediate value operand */
485 k = inst->k;
486
487 /* compute offsets only in the fake pass */
488 if (ctx->target == NULL)
489 ctx->offsets[i] = ctx->idx * 4;
490
491 switch (inst->code) {
492 case BPF_S_LD_IMM:
493 emit_mov_i(r_A, k, ctx);
494 break;
495 case BPF_S_LD_W_LEN:
496 ctx->seen |= SEEN_SKB;
497 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
498 emit(ARM_LDR_I(r_A, r_skb,
499 offsetof(struct sk_buff, len)), ctx);
500 break;
501 case BPF_S_LD_MEM:
502 /* A = scratch[k] */
503 ctx->seen |= SEEN_MEM_WORD(k);
504 emit(ARM_LDR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx);
505 break;
506 case BPF_S_LD_W_ABS:
507 load_order = 2;
508 goto load;
509 case BPF_S_LD_H_ABS:
510 load_order = 1;
511 goto load;
512 case BPF_S_LD_B_ABS:
513 load_order = 0;
514load:
515 /* the interpreter will deal with the negative K */
516 if ((int)k < 0)
517 return -ENOTSUPP;
518 emit_mov_i(r_off, k, ctx);
519load_common:
520 ctx->seen |= SEEN_DATA | SEEN_CALL;
521
522 if (load_order > 0) {
523 emit(ARM_SUB_I(r_scratch, r_skb_hl,
524 1 << load_order), ctx);
525 emit(ARM_CMP_R(r_scratch, r_off), ctx);
526 condt = ARM_COND_HS;
527 } else {
528 emit(ARM_CMP_R(r_skb_hl, r_off), ctx);
529 condt = ARM_COND_HI;
530 }
531
532 _emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data),
533 ctx);
534
535 if (load_order == 0)
536 _emit(condt, ARM_LDRB_I(r_A, r_scratch, 0),
537 ctx);
538 else if (load_order == 1)
539 emit_load_be16(condt, r_A, r_scratch, ctx);
540 else if (load_order == 2)
541 emit_load_be32(condt, r_A, r_scratch, ctx);
542
543 _emit(condt, ARM_B(b_imm(i + 1, ctx)), ctx);
544
545 /* the slowpath */
546 emit_mov_i(ARM_R3, (u32)load_func[load_order], ctx);
547 emit(ARM_MOV_R(ARM_R0, r_skb), ctx);
548 /* the offset is already in R1 */
549 emit_blx_r(ARM_R3, ctx);
550 /* check the result of skb_copy_bits */
551 emit(ARM_CMP_I(ARM_R1, 0), ctx);
552 emit_err_ret(ARM_COND_NE, ctx);
553 emit(ARM_MOV_R(r_A, ARM_R0), ctx);
554 break;
555 case BPF_S_LD_W_IND:
556 load_order = 2;
557 goto load_ind;
558 case BPF_S_LD_H_IND:
559 load_order = 1;
560 goto load_ind;
561 case BPF_S_LD_B_IND:
562 load_order = 0;
563load_ind:
564 OP_IMM3(ARM_ADD, r_off, r_X, k, ctx);
565 goto load_common;
566 case BPF_S_LDX_IMM:
567 ctx->seen |= SEEN_X;
568 emit_mov_i(r_X, k, ctx);
569 break;
570 case BPF_S_LDX_W_LEN:
571 ctx->seen |= SEEN_X | SEEN_SKB;
572 emit(ARM_LDR_I(r_X, r_skb,
573 offsetof(struct sk_buff, len)), ctx);
574 break;
575 case BPF_S_LDX_MEM:
576 ctx->seen |= SEEN_X | SEEN_MEM_WORD(k);
577 emit(ARM_LDR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx);
578 break;
579 case BPF_S_LDX_B_MSH:
580 /* x = ((*(frame + k)) & 0xf) << 2; */
581 ctx->seen |= SEEN_X | SEEN_DATA | SEEN_CALL;
582 /* the interpreter should deal with the negative K */
583 if ((int)k < 0)
584 return -1;
585 /* offset in r1: we might have to take the slow path */
586 emit_mov_i(r_off, k, ctx);
587 emit(ARM_CMP_R(r_skb_hl, r_off), ctx);
588
589 /* load in r0: common with the slowpath */
590 _emit(ARM_COND_HI, ARM_LDRB_R(ARM_R0, r_skb_data,
591 ARM_R1), ctx);
592 /*
593 * emit_mov_i() might generate one or two instructions,
594 * the same holds for emit_blx_r()
595 */
596 _emit(ARM_COND_HI, ARM_B(b_imm(i + 1, ctx) - 2), ctx);
597
598 emit(ARM_MOV_R(ARM_R0, r_skb), ctx);
599 /* r_off is r1 */
600 emit_mov_i(ARM_R3, (u32)jit_get_skb_b, ctx);
601 emit_blx_r(ARM_R3, ctx);
602 /* check the return value of skb_copy_bits */
603 emit(ARM_CMP_I(ARM_R1, 0), ctx);
604 emit_err_ret(ARM_COND_NE, ctx);
605
606 emit(ARM_AND_I(r_X, ARM_R0, 0x00f), ctx);
607 emit(ARM_LSL_I(r_X, r_X, 2), ctx);
608 break;
609 case BPF_S_ST:
610 ctx->seen |= SEEN_MEM_WORD(k);
611 emit(ARM_STR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx);
612 break;
613 case BPF_S_STX:
614 update_on_xread(ctx);
615 ctx->seen |= SEEN_MEM_WORD(k);
616 emit(ARM_STR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx);
617 break;
618 case BPF_S_ALU_ADD_K:
619 /* A += K */
620 OP_IMM3(ARM_ADD, r_A, r_A, k, ctx);
621 break;
622 case BPF_S_ALU_ADD_X:
623 update_on_xread(ctx);
624 emit(ARM_ADD_R(r_A, r_A, r_X), ctx);
625 break;
626 case BPF_S_ALU_SUB_K:
627 /* A -= K */
628 OP_IMM3(ARM_SUB, r_A, r_A, k, ctx);
629 break;
630 case BPF_S_ALU_SUB_X:
631 update_on_xread(ctx);
632 emit(ARM_SUB_R(r_A, r_A, r_X), ctx);
633 break;
634 case BPF_S_ALU_MUL_K:
635 /* A *= K */
636 emit_mov_i(r_scratch, k, ctx);
637 emit(ARM_MUL(r_A, r_A, r_scratch), ctx);
638 break;
639 case BPF_S_ALU_MUL_X:
640 update_on_xread(ctx);
641 emit(ARM_MUL(r_A, r_A, r_X), ctx);
642 break;
643 case BPF_S_ALU_DIV_K:
644 if (k == 1)
645 break;
646 emit_mov_i(r_scratch, k, ctx);
647 emit_udiv(r_A, r_A, r_scratch, ctx);
648 break;
649 case BPF_S_ALU_DIV_X:
650 update_on_xread(ctx);
651 emit(ARM_CMP_I(r_X, 0), ctx);
652 emit_err_ret(ARM_COND_EQ, ctx);
653 emit_udiv(r_A, r_A, r_X, ctx);
654 break;
655 case BPF_S_ALU_OR_K:
656 /* A |= K */
657 OP_IMM3(ARM_ORR, r_A, r_A, k, ctx);
658 break;
659 case BPF_S_ALU_OR_X:
660 update_on_xread(ctx);
661 emit(ARM_ORR_R(r_A, r_A, r_X), ctx);
662 break;
663 case BPF_S_ALU_XOR_K:
664 /* A ^= K; */
665 OP_IMM3(ARM_EOR, r_A, r_A, k, ctx);
666 break;
667 case BPF_S_ANC_ALU_XOR_X:
668 case BPF_S_ALU_XOR_X:
669 /* A ^= X */
670 update_on_xread(ctx);
671 emit(ARM_EOR_R(r_A, r_A, r_X), ctx);
672 break;
673 case BPF_S_ALU_AND_K:
674 /* A &= K */
675 OP_IMM3(ARM_AND, r_A, r_A, k, ctx);
676 break;
677 case BPF_S_ALU_AND_X:
678 update_on_xread(ctx);
679 emit(ARM_AND_R(r_A, r_A, r_X), ctx);
680 break;
681 case BPF_S_ALU_LSH_K:
682 if (unlikely(k > 31))
683 return -1;
684 emit(ARM_LSL_I(r_A, r_A, k), ctx);
685 break;
686 case BPF_S_ALU_LSH_X:
687 update_on_xread(ctx);
688 emit(ARM_LSL_R(r_A, r_A, r_X), ctx);
689 break;
690 case BPF_S_ALU_RSH_K:
691 if (unlikely(k > 31))
692 return -1;
693 emit(ARM_LSR_I(r_A, r_A, k), ctx);
694 break;
695 case BPF_S_ALU_RSH_X:
696 update_on_xread(ctx);
697 emit(ARM_LSR_R(r_A, r_A, r_X), ctx);
698 break;
699 case BPF_S_ALU_NEG:
700 /* A = -A */
701 emit(ARM_RSB_I(r_A, r_A, 0), ctx);
702 break;
703 case BPF_S_JMP_JA:
704 /* pc += K */
705 emit(ARM_B(b_imm(i + k + 1, ctx)), ctx);
706 break;
707 case BPF_S_JMP_JEQ_K:
708 /* pc += (A == K) ? pc->jt : pc->jf */
709 condt = ARM_COND_EQ;
710 goto cmp_imm;
711 case BPF_S_JMP_JGT_K:
712 /* pc += (A > K) ? pc->jt : pc->jf */
713 condt = ARM_COND_HI;
714 goto cmp_imm;
715 case BPF_S_JMP_JGE_K:
716 /* pc += (A >= K) ? pc->jt : pc->jf */
717 condt = ARM_COND_HS;
718cmp_imm:
719 imm12 = imm8m(k);
720 if (imm12 < 0) {
721 emit_mov_i_no8m(r_scratch, k, ctx);
722 emit(ARM_CMP_R(r_A, r_scratch), ctx);
723 } else {
724 emit(ARM_CMP_I(r_A, imm12), ctx);
725 }
726cond_jump:
727 if (inst->jt)
728 _emit(condt, ARM_B(b_imm(i + inst->jt + 1,
729 ctx)), ctx);
730 if (inst->jf)
731 _emit(condt ^ 1, ARM_B(b_imm(i + inst->jf + 1,
732 ctx)), ctx);
733 break;
734 case BPF_S_JMP_JEQ_X:
735 /* pc += (A == X) ? pc->jt : pc->jf */
736 condt = ARM_COND_EQ;
737 goto cmp_x;
738 case BPF_S_JMP_JGT_X:
739 /* pc += (A > X) ? pc->jt : pc->jf */
740 condt = ARM_COND_HI;
741 goto cmp_x;
742 case BPF_S_JMP_JGE_X:
743 /* pc += (A >= X) ? pc->jt : pc->jf */
744 condt = ARM_COND_CS;
745cmp_x:
746 update_on_xread(ctx);
747 emit(ARM_CMP_R(r_A, r_X), ctx);
748 goto cond_jump;
749 case BPF_S_JMP_JSET_K:
750 /* pc += (A & K) ? pc->jt : pc->jf */
751 condt = ARM_COND_NE;
752 /* not set iff all zeroes iff Z==1 iff EQ */
753
754 imm12 = imm8m(k);
755 if (imm12 < 0) {
756 emit_mov_i_no8m(r_scratch, k, ctx);
757 emit(ARM_TST_R(r_A, r_scratch), ctx);
758 } else {
759 emit(ARM_TST_I(r_A, imm12), ctx);
760 }
761 goto cond_jump;
762 case BPF_S_JMP_JSET_X:
763 /* pc += (A & X) ? pc->jt : pc->jf */
764 update_on_xread(ctx);
765 condt = ARM_COND_NE;
766 emit(ARM_TST_R(r_A, r_X), ctx);
767 goto cond_jump;
768 case BPF_S_RET_A:
769 emit(ARM_MOV_R(ARM_R0, r_A), ctx);
770 goto b_epilogue;
771 case BPF_S_RET_K:
772 if ((k == 0) && (ctx->ret0_fp_idx < 0))
773 ctx->ret0_fp_idx = i;
774 emit_mov_i(ARM_R0, k, ctx);
775b_epilogue:
776 if (i != ctx->skf->len - 1)
777 emit(ARM_B(b_imm(prog->len, ctx)), ctx);
778 break;
779 case BPF_S_MISC_TAX:
780 /* X = A */
781 ctx->seen |= SEEN_X;
782 emit(ARM_MOV_R(r_X, r_A), ctx);
783 break;
784 case BPF_S_MISC_TXA:
785 /* A = X */
786 update_on_xread(ctx);
787 emit(ARM_MOV_R(r_A, r_X), ctx);
788 break;
789 case BPF_S_ANC_PROTOCOL:
790 /* A = ntohs(skb->protocol) */
791 ctx->seen |= SEEN_SKB;
792 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
793 protocol) != 2);
794 off = offsetof(struct sk_buff, protocol);
795 emit(ARM_LDRH_I(r_scratch, r_skb, off), ctx);
796 emit_swap16(r_A, r_scratch, ctx);
797 break;
798 case BPF_S_ANC_CPU:
799 /* r_scratch = current_thread_info() */
800 OP_IMM3(ARM_BIC, r_scratch, ARM_SP, THREAD_SIZE - 1, ctx);
801 /* A = current_thread_info()->cpu */
802 BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, cpu) != 4);
803 off = offsetof(struct thread_info, cpu);
804 emit(ARM_LDR_I(r_A, r_scratch, off), ctx);
805 break;
806 case BPF_S_ANC_IFINDEX:
807 /* A = skb->dev->ifindex */
808 ctx->seen |= SEEN_SKB;
809 off = offsetof(struct sk_buff, dev);
810 emit(ARM_LDR_I(r_scratch, r_skb, off), ctx);
811
812 emit(ARM_CMP_I(r_scratch, 0), ctx);
813 emit_err_ret(ARM_COND_EQ, ctx);
814
815 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device,
816 ifindex) != 4);
817 off = offsetof(struct net_device, ifindex);
818 emit(ARM_LDR_I(r_A, r_scratch, off), ctx);
819 break;
820 case BPF_S_ANC_MARK:
821 ctx->seen |= SEEN_SKB;
822 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
823 off = offsetof(struct sk_buff, mark);
824 emit(ARM_LDR_I(r_A, r_skb, off), ctx);
825 break;
826 case BPF_S_ANC_RXHASH:
827 ctx->seen |= SEEN_SKB;
828 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
829 off = offsetof(struct sk_buff, hash);
830 emit(ARM_LDR_I(r_A, r_skb, off), ctx);
831 break;
832 case BPF_S_ANC_VLAN_TAG:
833 case BPF_S_ANC_VLAN_TAG_PRESENT:
834 ctx->seen |= SEEN_SKB;
835 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
836 off = offsetof(struct sk_buff, vlan_tci);
837 emit(ARM_LDRH_I(r_A, r_skb, off), ctx);
838 if (inst->code == BPF_S_ANC_VLAN_TAG)
839 OP_IMM3(ARM_AND, r_A, r_A, VLAN_VID_MASK, ctx);
840 else
841 OP_IMM3(ARM_AND, r_A, r_A, VLAN_TAG_PRESENT, ctx);
842 break;
843 case BPF_S_ANC_QUEUE:
844 ctx->seen |= SEEN_SKB;
845 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
846 queue_mapping) != 2);
847 BUILD_BUG_ON(offsetof(struct sk_buff,
848 queue_mapping) > 0xff);
849 off = offsetof(struct sk_buff, queue_mapping);
850 emit(ARM_LDRH_I(r_A, r_skb, off), ctx);
851 break;
852 default:
853 return -1;
854 }
855 }
856
857 /* compute offsets only during the first pass */
858 if (ctx->target == NULL)
859 ctx->offsets[i] = ctx->idx * 4;
860
861 return 0;
862}
863
864
865void bpf_jit_compile(struct sk_filter *fp)
866{
867 struct jit_ctx ctx;
868 unsigned tmp_idx;
869 unsigned alloc_size;
870
871 if (!bpf_jit_enable)
872 return;
873
874 memset(&ctx, 0, sizeof(ctx));
875 ctx.skf = fp;
876 ctx.ret0_fp_idx = -1;
877
878 ctx.offsets = kzalloc(4 * (ctx.skf->len + 1), GFP_KERNEL);
879 if (ctx.offsets == NULL)
880 return;
881
882 /* fake pass to fill in the ctx->seen */
883 if (unlikely(build_body(&ctx)))
884 goto out;
885
886 tmp_idx = ctx.idx;
887 build_prologue(&ctx);
888 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
889
890#if __LINUX_ARM_ARCH__ < 7
891 tmp_idx = ctx.idx;
892 build_epilogue(&ctx);
893 ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4;
894
895 ctx.idx += ctx.imm_count;
896 if (ctx.imm_count) {
897 ctx.imms = kzalloc(4 * ctx.imm_count, GFP_KERNEL);
898 if (ctx.imms == NULL)
899 goto out;
900 }
901#else
902 /* there's nothing after the epilogue on ARMv7 */
903 build_epilogue(&ctx);
904#endif
905
906 alloc_size = 4 * ctx.idx;
907 ctx.target = module_alloc(alloc_size);
908 if (unlikely(ctx.target == NULL))
909 goto out;
910
911 ctx.idx = 0;
912 build_prologue(&ctx);
913 build_body(&ctx);
914 build_epilogue(&ctx);
915
916 flush_icache_range((u32)ctx.target, (u32)(ctx.target + ctx.idx));
917
918#if __LINUX_ARM_ARCH__ < 7
919 if (ctx.imm_count)
920 kfree(ctx.imms);
921#endif
922
923 if (bpf_jit_enable > 1)
924 /* there are 2 passes here */
925 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
926
927 fp->bpf_func = (void *)ctx.target;
928 fp->jited = 1;
929out:
930 kfree(ctx.offsets);
931 return;
932}
933
934void bpf_jit_free(struct sk_filter *fp)
935{
936 if (fp->jited)
937 module_free(NULL, fp->bpf_func);
938 kfree(fp);
939}