Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Just-In-Time compiler for eBPF bytecode on MIPS.
4 * Implementation of JIT functions for 64-bit CPUs.
5 *
6 * Copyright (c) 2021 Anyfi Networks AB.
7 * Author: Johan Almbladh <johan.almbladh@gmail.com>
8 *
9 * Based on code and ideas from
10 * Copyright (c) 2017 Cavium, Inc.
11 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
12 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
13 */
14
15#include <linux/errno.h>
16#include <linux/filter.h>
17#include <linux/bpf.h>
18#include <asm/cpu-features.h>
19#include <asm/isa-rev.h>
20#include <asm/uasm.h>
21
22#include "bpf_jit_comp.h"
23
24/* MIPS t0-t3 are not available in the n64 ABI */
25#undef MIPS_R_T0
26#undef MIPS_R_T1
27#undef MIPS_R_T2
28#undef MIPS_R_T3
29
30/* Stack is 16-byte aligned in n64 ABI */
31#define MIPS_STACK_ALIGNMENT 16
32
33/* Extra 64-bit eBPF registers used by JIT */
34#define JIT_REG_TC (MAX_BPF_JIT_REG + 0)
35#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1)
36
37/* Number of prologue bytes to skip when doing a tail call */
38#define JIT_TCALL_SKIP 4
39
40/* Callee-saved CPU registers that the JIT must preserve */
41#define JIT_CALLEE_REGS \
42 (BIT(MIPS_R_S0) | \
43 BIT(MIPS_R_S1) | \
44 BIT(MIPS_R_S2) | \
45 BIT(MIPS_R_S3) | \
46 BIT(MIPS_R_S4) | \
47 BIT(MIPS_R_S5) | \
48 BIT(MIPS_R_S6) | \
49 BIT(MIPS_R_S7) | \
50 BIT(MIPS_R_GP) | \
51 BIT(MIPS_R_FP) | \
52 BIT(MIPS_R_RA))
53
54/* Caller-saved CPU registers available for JIT use */
55#define JIT_CALLER_REGS \
56 (BIT(MIPS_R_A5) | \
57 BIT(MIPS_R_A6) | \
58 BIT(MIPS_R_A7))
59/*
60 * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers.
61 * MIPS registers t4 - t7 may be used by the JIT as temporary registers.
62 * MIPS registers t8 - t9 are reserved for single-register common functions.
63 */
64static const u8 bpf2mips64[] = {
65 /* Return value from in-kernel function, and exit value from eBPF */
66 [BPF_REG_0] = MIPS_R_V0,
67 /* Arguments from eBPF program to in-kernel function */
68 [BPF_REG_1] = MIPS_R_A0,
69 [BPF_REG_2] = MIPS_R_A1,
70 [BPF_REG_3] = MIPS_R_A2,
71 [BPF_REG_4] = MIPS_R_A3,
72 [BPF_REG_5] = MIPS_R_A4,
73 /* Callee-saved registers that in-kernel function will preserve */
74 [BPF_REG_6] = MIPS_R_S0,
75 [BPF_REG_7] = MIPS_R_S1,
76 [BPF_REG_8] = MIPS_R_S2,
77 [BPF_REG_9] = MIPS_R_S3,
78 /* Read-only frame pointer to access the eBPF stack */
79 [BPF_REG_FP] = MIPS_R_FP,
80 /* Temporary register for blinding constants */
81 [BPF_REG_AX] = MIPS_R_AT,
82 /* Tail call count register, caller-saved */
83 [JIT_REG_TC] = MIPS_R_A5,
84 /* Constant for register zero-extension */
85 [JIT_REG_ZX] = MIPS_R_V1,
86};
87
88/*
89 * MIPS 32-bit operations on 64-bit registers generate a sign-extended
90 * result. However, the eBPF ISA mandates zero-extension, so we rely on the
91 * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic
92 * operations, right shift and byte swap require properly sign-extended
93 * operands or the result is unpredictable. We emit explicit sign-extensions
94 * in those cases.
95 */
96
97/* Sign extension */
98static void emit_sext(struct jit_context *ctx, u8 dst, u8 src)
99{
100 emit(ctx, sll, dst, src, 0);
101 clobber_reg(ctx, dst);
102}
103
104/* Zero extension */
105static void emit_zext(struct jit_context *ctx, u8 dst)
106{
107 if (cpu_has_mips64r2 || cpu_has_mips64r6) {
108 emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
109 } else {
110 emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]);
111 access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */
112 }
113 clobber_reg(ctx, dst);
114}
115
116/* Zero extension, if verifier does not do it for us */
117static void emit_zext_ver(struct jit_context *ctx, u8 dst)
118{
119 if (!ctx->program->aux->verifier_zext)
120 emit_zext(ctx, dst);
121}
122
123/* dst = imm (64-bit) */
124static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64)
125{
126 if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) {
127 emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64);
128 } else if (imm64 >= 0xffffffff80000000ULL ||
129 (imm64 < 0x80000000 && imm64 > 0xffff)) {
130 emit(ctx, lui, dst, (s16)(imm64 >> 16));
131 emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff);
132 } else {
133 u8 acc = MIPS_R_ZERO;
134 int shift = 0;
135 int k;
136
137 for (k = 0; k < 4; k++) {
138 u16 half = imm64 >> (48 - 16 * k);
139
140 if (acc == dst)
141 shift += 16;
142
143 if (half) {
144 if (shift)
145 emit(ctx, dsll_safe, dst, dst, shift);
146 emit(ctx, ori, dst, acc, half);
147 acc = dst;
148 shift = 0;
149 }
150 }
151 if (shift)
152 emit(ctx, dsll_safe, dst, dst, shift);
153 }
154 clobber_reg(ctx, dst);
155}
156
157/* ALU immediate operation (64-bit) */
158static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
159{
160 switch (BPF_OP(op)) {
161 /* dst = dst | imm */
162 case BPF_OR:
163 emit(ctx, ori, dst, dst, (u16)imm);
164 break;
165 /* dst = dst ^ imm */
166 case BPF_XOR:
167 emit(ctx, xori, dst, dst, (u16)imm);
168 break;
169 /* dst = -dst */
170 case BPF_NEG:
171 emit(ctx, dsubu, dst, MIPS_R_ZERO, dst);
172 break;
173 /* dst = dst << imm */
174 case BPF_LSH:
175 emit(ctx, dsll_safe, dst, dst, imm);
176 break;
177 /* dst = dst >> imm */
178 case BPF_RSH:
179 emit(ctx, dsrl_safe, dst, dst, imm);
180 break;
181 /* dst = dst >> imm (arithmetic) */
182 case BPF_ARSH:
183 emit(ctx, dsra_safe, dst, dst, imm);
184 break;
185 /* dst = dst + imm */
186 case BPF_ADD:
187 emit(ctx, daddiu, dst, dst, imm);
188 break;
189 /* dst = dst - imm */
190 case BPF_SUB:
191 emit(ctx, daddiu, dst, dst, -imm);
192 break;
193 default:
194 /* Width-generic operations */
195 emit_alu_i(ctx, dst, imm, op);
196 }
197 clobber_reg(ctx, dst);
198}
199
200/* ALU register operation (64-bit) */
201static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
202{
203 switch (BPF_OP(op)) {
204 /* dst = dst << src */
205 case BPF_LSH:
206 emit(ctx, dsllv, dst, dst, src);
207 break;
208 /* dst = dst >> src */
209 case BPF_RSH:
210 emit(ctx, dsrlv, dst, dst, src);
211 break;
212 /* dst = dst >> src (arithmetic) */
213 case BPF_ARSH:
214 emit(ctx, dsrav, dst, dst, src);
215 break;
216 /* dst = dst + src */
217 case BPF_ADD:
218 emit(ctx, daddu, dst, dst, src);
219 break;
220 /* dst = dst - src */
221 case BPF_SUB:
222 emit(ctx, dsubu, dst, dst, src);
223 break;
224 /* dst = dst * src */
225 case BPF_MUL:
226 if (cpu_has_mips64r6) {
227 emit(ctx, dmulu, dst, dst, src);
228 } else {
229 emit(ctx, dmultu, dst, src);
230 emit(ctx, mflo, dst);
231 }
232 break;
233 /* dst = dst / src */
234 case BPF_DIV:
235 if (cpu_has_mips64r6) {
236 emit(ctx, ddivu_r6, dst, dst, src);
237 } else {
238 emit(ctx, ddivu, dst, src);
239 emit(ctx, mflo, dst);
240 }
241 break;
242 /* dst = dst % src */
243 case BPF_MOD:
244 if (cpu_has_mips64r6) {
245 emit(ctx, dmodu, dst, dst, src);
246 } else {
247 emit(ctx, ddivu, dst, src);
248 emit(ctx, mfhi, dst);
249 }
250 break;
251 default:
252 /* Width-generic operations */
253 emit_alu_r(ctx, dst, src, op);
254 }
255 clobber_reg(ctx, dst);
256}
257
258/* Swap sub words in a register double word */
259static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits)
260{
261 u8 tmp = MIPS_R_T9;
262
263 emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */
264 emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */
265 emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */
266 emit(ctx, and, dst, dst, mask); /* dst = dst & mask */
267 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */
268}
269
270/* Swap bytes and truncate a register double word, word or half word */
271static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width)
272{
273 switch (width) {
274 /* Swap bytes in a double word */
275 case 64:
276 if (cpu_has_mips64r2 || cpu_has_mips64r6) {
277 emit(ctx, dsbh, dst, dst);
278 emit(ctx, dshd, dst, dst);
279 } else {
280 u8 t1 = MIPS_R_T6;
281 u8 t2 = MIPS_R_T7;
282
283 emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */
284 emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */
285 emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */
286
287 emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff);
288 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
289 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
290 emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */
291
292 emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */
293 emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */
294 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
295 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
296 emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */
297 }
298 break;
299 /* Swap bytes in a half word */
300 /* Swap bytes in a word */
301 case 32:
302 case 16:
303 emit_sext(ctx, dst, dst);
304 emit_bswap_r(ctx, dst, width);
305 if (cpu_has_mips64r2 || cpu_has_mips64r6)
306 emit_zext(ctx, dst);
307 break;
308 }
309 clobber_reg(ctx, dst);
310}
311
312/* Truncate a register double word, word or half word */
313static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width)
314{
315 switch (width) {
316 case 64:
317 break;
318 /* Zero-extend a word */
319 case 32:
320 emit_zext(ctx, dst);
321 break;
322 /* Zero-extend a half word */
323 case 16:
324 emit(ctx, andi, dst, dst, 0xffff);
325 break;
326 }
327 clobber_reg(ctx, dst);
328}
329
330/* Load operation: dst = *(size*)(src + off) */
331static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
332{
333 switch (size) {
334 /* Load a byte */
335 case BPF_B:
336 emit(ctx, lbu, dst, off, src);
337 break;
338 /* Load a half word */
339 case BPF_H:
340 emit(ctx, lhu, dst, off, src);
341 break;
342 /* Load a word */
343 case BPF_W:
344 emit(ctx, lwu, dst, off, src);
345 break;
346 /* Load a double word */
347 case BPF_DW:
348 emit(ctx, ld, dst, off, src);
349 break;
350 }
351 clobber_reg(ctx, dst);
352}
353
354/* Store operation: *(size *)(dst + off) = src */
355static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
356{
357 switch (size) {
358 /* Store a byte */
359 case BPF_B:
360 emit(ctx, sb, src, off, dst);
361 break;
362 /* Store a half word */
363 case BPF_H:
364 emit(ctx, sh, src, off, dst);
365 break;
366 /* Store a word */
367 case BPF_W:
368 emit(ctx, sw, src, off, dst);
369 break;
370 /* Store a double word */
371 case BPF_DW:
372 emit(ctx, sd, src, off, dst);
373 break;
374 }
375}
376
377/* Atomic read-modify-write */
378static void emit_atomic_r64(struct jit_context *ctx,
379 u8 dst, u8 src, s16 off, u8 code)
380{
381 u8 t1 = MIPS_R_T6;
382 u8 t2 = MIPS_R_T7;
383
384 LLSC_sync(ctx);
385 emit(ctx, lld, t1, off, dst);
386 switch (code) {
387 case BPF_ADD:
388 case BPF_ADD | BPF_FETCH:
389 emit(ctx, daddu, t2, t1, src);
390 break;
391 case BPF_AND:
392 case BPF_AND | BPF_FETCH:
393 emit(ctx, and, t2, t1, src);
394 break;
395 case BPF_OR:
396 case BPF_OR | BPF_FETCH:
397 emit(ctx, or, t2, t1, src);
398 break;
399 case BPF_XOR:
400 case BPF_XOR | BPF_FETCH:
401 emit(ctx, xor, t2, t1, src);
402 break;
403 case BPF_XCHG:
404 emit(ctx, move, t2, src);
405 break;
406 }
407 emit(ctx, scd, t2, off, dst);
408 emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
409 emit(ctx, nop); /* Delay slot */
410
411 if (code & BPF_FETCH) {
412 emit(ctx, move, src, t1);
413 clobber_reg(ctx, src);
414 }
415}
416
417/* Atomic compare-and-exchange */
418static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
419{
420 u8 r0 = bpf2mips64[BPF_REG_0];
421 u8 t1 = MIPS_R_T6;
422 u8 t2 = MIPS_R_T7;
423
424 LLSC_sync(ctx);
425 emit(ctx, lld, t1, off, dst);
426 emit(ctx, bne, t1, r0, 12);
427 emit(ctx, move, t2, src); /* Delay slot */
428 emit(ctx, scd, t2, off, dst);
429 emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
430 emit(ctx, move, r0, t1); /* Delay slot */
431
432 clobber_reg(ctx, r0);
433}
434
435/* Function call */
436static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
437{
438 u8 zx = bpf2mips64[JIT_REG_ZX];
439 u8 tmp = MIPS_R_T6;
440 bool fixed;
441 u64 addr;
442
443 /* Decode the call address */
444 if (bpf_jit_get_func_addr(ctx->program, insn, false,
445 &addr, &fixed) < 0)
446 return -1;
447 if (!fixed)
448 return -1;
449
450 /* Push caller-saved registers on stack */
451 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
452
453 /* Emit function call */
454 emit_mov_i64(ctx, tmp, addr & JALR_MASK);
455 emit(ctx, jalr, MIPS_R_RA, tmp);
456 emit(ctx, nop); /* Delay slot */
457
458 /* Restore caller-saved registers */
459 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
460
461 /* Re-initialize the JIT zero-extension register if accessed */
462 if (ctx->accessed & BIT(JIT_REG_ZX)) {
463 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
464 emit(ctx, dsrl32, zx, zx, 0);
465 }
466
467 clobber_reg(ctx, MIPS_R_RA);
468 clobber_reg(ctx, MIPS_R_V0);
469 clobber_reg(ctx, MIPS_R_V1);
470 return 0;
471}
472
473/* Function tail call */
474static int emit_tail_call(struct jit_context *ctx)
475{
476 u8 ary = bpf2mips64[BPF_REG_2];
477 u8 ind = bpf2mips64[BPF_REG_3];
478 u8 tcc = bpf2mips64[JIT_REG_TC];
479 u8 tmp = MIPS_R_T6;
480 int off;
481
482 /*
483 * Tail call:
484 * eBPF R1 - function argument (context ptr), passed in a0-a1
485 * eBPF R2 - ptr to object with array of function entry points
486 * eBPF R3 - array index of function to be called
487 */
488
489 /* if (ind >= ary->map.max_entries) goto out */
490 off = offsetof(struct bpf_array, map.max_entries);
491 if (off > 0x7fff)
492 return -1;
493 emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/
494 emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */
495 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
496
497 /* if (--TCC < 0) goto out */
498 emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */
499 emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */
500 /* (next insn delay slot) */
501 /* prog = ary->ptrs[ind] */
502 off = offsetof(struct bpf_array, ptrs);
503 if (off > 0x7fff)
504 return -1;
505 emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */
506 emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */
507 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
508
509 /* if (prog == 0) goto out */
510 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
511 emit(ctx, nop); /* Delay slot */
512
513 /* func = prog->bpf_func + 8 (prologue skip offset) */
514 off = offsetof(struct bpf_prog, bpf_func);
515 if (off > 0x7fff)
516 return -1;
517 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
518 emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */
519
520 /* goto func */
521 build_epilogue(ctx, tmp);
522 access_reg(ctx, JIT_REG_TC);
523 return 0;
524}
525
526/*
527 * Stack frame layout for a JITed program (stack grows down).
528 *
529 * Higher address : Previous stack frame :
530 * +===========================+ <--- MIPS sp before call
531 * | Callee-saved registers, |
532 * | including RA and FP |
533 * +---------------------------+ <--- eBPF FP (MIPS fp)
534 * | Local eBPF variables |
535 * | allocated by program |
536 * +---------------------------+
537 * | Reserved for caller-saved |
538 * | registers |
539 * Lower address +===========================+ <--- MIPS sp
540 */
541
542/* Build program prologue to set up the stack and registers */
543void build_prologue(struct jit_context *ctx)
544{
545 u8 fp = bpf2mips64[BPF_REG_FP];
546 u8 tc = bpf2mips64[JIT_REG_TC];
547 u8 zx = bpf2mips64[JIT_REG_ZX];
548 int stack, saved, locals, reserved;
549
550 /*
551 * In the unlikely event that the TCC limit is raised to more
552 * than 16 bits, it is clamped to the maximum value allowed for
553 * the generated code (0xffff). It is better fail to compile
554 * instead of degrading gracefully.
555 */
556 BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
557
558 /*
559 * The first instruction initializes the tail call count register.
560 * On a tail call, the calling function jumps into the prologue
561 * after this instruction.
562 */
563 emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
564
565 /* === Entry-point for tail calls === */
566
567 /*
568 * If the eBPF frame pointer and tail call count registers were
569 * accessed they must be preserved. Mark them as clobbered here
570 * to save and restore them on the stack as needed.
571 */
572 if (ctx->accessed & BIT(BPF_REG_FP))
573 clobber_reg(ctx, fp);
574 if (ctx->accessed & BIT(JIT_REG_TC))
575 clobber_reg(ctx, tc);
576 if (ctx->accessed & BIT(JIT_REG_ZX))
577 clobber_reg(ctx, zx);
578
579 /* Compute the stack space needed for callee-saved registers */
580 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64);
581 saved = ALIGN(saved, MIPS_STACK_ALIGNMENT);
582
583 /* Stack space used by eBPF program local data */
584 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT);
585
586 /*
587 * If we are emitting function calls, reserve extra stack space for
588 * caller-saved registers needed by the JIT. The required space is
589 * computed automatically during resource usage discovery (pass 1).
590 */
591 reserved = ctx->stack_used;
592
593 /* Allocate the stack frame */
594 stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT);
595 if (stack)
596 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack);
597
598 /* Store callee-saved registers on stack */
599 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved);
600
601 /* Initialize the eBPF frame pointer if accessed */
602 if (ctx->accessed & BIT(BPF_REG_FP))
603 emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved);
604
605 /* Initialize the ePF JIT zero-extension register if accessed */
606 if (ctx->accessed & BIT(JIT_REG_ZX)) {
607 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
608 emit(ctx, dsrl32, zx, zx, 0);
609 }
610
611 ctx->saved_size = saved;
612 ctx->stack_size = stack;
613}
614
615/* Build the program epilogue to restore the stack and registers */
616void build_epilogue(struct jit_context *ctx, int dest_reg)
617{
618 /* Restore callee-saved registers from stack */
619 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0,
620 ctx->stack_size - ctx->saved_size);
621
622 /* Release the stack frame */
623 if (ctx->stack_size)
624 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size);
625
626 /* Jump to return address and sign-extend the 32-bit return value */
627 emit(ctx, jr, dest_reg);
628 emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */
629}
630
631/* Build one eBPF instruction */
632int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
633{
634 u8 dst = bpf2mips64[insn->dst_reg];
635 u8 src = bpf2mips64[insn->src_reg];
636 u8 res = bpf2mips64[BPF_REG_0];
637 u8 code = insn->code;
638 s16 off = insn->off;
639 s32 imm = insn->imm;
640 s32 val, rel;
641 u8 alu, jmp;
642
643 switch (code) {
644 /* ALU operations */
645 /* dst = imm */
646 case BPF_ALU | BPF_MOV | BPF_K:
647 emit_mov_i(ctx, dst, imm);
648 emit_zext_ver(ctx, dst);
649 break;
650 /* dst = src */
651 case BPF_ALU | BPF_MOV | BPF_X:
652 if (imm == 1) {
653 /* Special mov32 for zext */
654 emit_zext(ctx, dst);
655 } else {
656 emit_mov_r(ctx, dst, src);
657 emit_zext_ver(ctx, dst);
658 }
659 break;
660 /* dst = -dst */
661 case BPF_ALU | BPF_NEG:
662 emit_sext(ctx, dst, dst);
663 emit_alu_i(ctx, dst, 0, BPF_NEG);
664 emit_zext_ver(ctx, dst);
665 break;
666 /* dst = dst & imm */
667 /* dst = dst | imm */
668 /* dst = dst ^ imm */
669 /* dst = dst << imm */
670 case BPF_ALU | BPF_OR | BPF_K:
671 case BPF_ALU | BPF_AND | BPF_K:
672 case BPF_ALU | BPF_XOR | BPF_K:
673 case BPF_ALU | BPF_LSH | BPF_K:
674 if (!valid_alu_i(BPF_OP(code), imm)) {
675 emit_mov_i(ctx, MIPS_R_T4, imm);
676 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
677 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
678 emit_alu_i(ctx, dst, val, alu);
679 }
680 emit_zext_ver(ctx, dst);
681 break;
682 /* dst = dst >> imm */
683 /* dst = dst >> imm (arithmetic) */
684 /* dst = dst + imm */
685 /* dst = dst - imm */
686 /* dst = dst * imm */
687 /* dst = dst / imm */
688 /* dst = dst % imm */
689 case BPF_ALU | BPF_RSH | BPF_K:
690 case BPF_ALU | BPF_ARSH | BPF_K:
691 case BPF_ALU | BPF_ADD | BPF_K:
692 case BPF_ALU | BPF_SUB | BPF_K:
693 case BPF_ALU | BPF_MUL | BPF_K:
694 case BPF_ALU | BPF_DIV | BPF_K:
695 case BPF_ALU | BPF_MOD | BPF_K:
696 if (!valid_alu_i(BPF_OP(code), imm)) {
697 emit_sext(ctx, dst, dst);
698 emit_mov_i(ctx, MIPS_R_T4, imm);
699 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
700 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
701 emit_sext(ctx, dst, dst);
702 emit_alu_i(ctx, dst, val, alu);
703 }
704 emit_zext_ver(ctx, dst);
705 break;
706 /* dst = dst & src */
707 /* dst = dst | src */
708 /* dst = dst ^ src */
709 /* dst = dst << src */
710 case BPF_ALU | BPF_AND | BPF_X:
711 case BPF_ALU | BPF_OR | BPF_X:
712 case BPF_ALU | BPF_XOR | BPF_X:
713 case BPF_ALU | BPF_LSH | BPF_X:
714 emit_alu_r(ctx, dst, src, BPF_OP(code));
715 emit_zext_ver(ctx, dst);
716 break;
717 /* dst = dst >> src */
718 /* dst = dst >> src (arithmetic) */
719 /* dst = dst + src */
720 /* dst = dst - src */
721 /* dst = dst * src */
722 /* dst = dst / src */
723 /* dst = dst % src */
724 case BPF_ALU | BPF_RSH | BPF_X:
725 case BPF_ALU | BPF_ARSH | BPF_X:
726 case BPF_ALU | BPF_ADD | BPF_X:
727 case BPF_ALU | BPF_SUB | BPF_X:
728 case BPF_ALU | BPF_MUL | BPF_X:
729 case BPF_ALU | BPF_DIV | BPF_X:
730 case BPF_ALU | BPF_MOD | BPF_X:
731 emit_sext(ctx, dst, dst);
732 emit_sext(ctx, MIPS_R_T4, src);
733 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
734 emit_zext_ver(ctx, dst);
735 break;
736 /* dst = imm (64-bit) */
737 case BPF_ALU64 | BPF_MOV | BPF_K:
738 emit_mov_i(ctx, dst, imm);
739 break;
740 /* dst = src (64-bit) */
741 case BPF_ALU64 | BPF_MOV | BPF_X:
742 emit_mov_r(ctx, dst, src);
743 break;
744 /* dst = -dst (64-bit) */
745 case BPF_ALU64 | BPF_NEG:
746 emit_alu_i64(ctx, dst, 0, BPF_NEG);
747 break;
748 /* dst = dst & imm (64-bit) */
749 /* dst = dst | imm (64-bit) */
750 /* dst = dst ^ imm (64-bit) */
751 /* dst = dst << imm (64-bit) */
752 /* dst = dst >> imm (64-bit) */
753 /* dst = dst >> imm ((64-bit, arithmetic) */
754 /* dst = dst + imm (64-bit) */
755 /* dst = dst - imm (64-bit) */
756 /* dst = dst * imm (64-bit) */
757 /* dst = dst / imm (64-bit) */
758 /* dst = dst % imm (64-bit) */
759 case BPF_ALU64 | BPF_AND | BPF_K:
760 case BPF_ALU64 | BPF_OR | BPF_K:
761 case BPF_ALU64 | BPF_XOR | BPF_K:
762 case BPF_ALU64 | BPF_LSH | BPF_K:
763 case BPF_ALU64 | BPF_RSH | BPF_K:
764 case BPF_ALU64 | BPF_ARSH | BPF_K:
765 case BPF_ALU64 | BPF_ADD | BPF_K:
766 case BPF_ALU64 | BPF_SUB | BPF_K:
767 case BPF_ALU64 | BPF_MUL | BPF_K:
768 case BPF_ALU64 | BPF_DIV | BPF_K:
769 case BPF_ALU64 | BPF_MOD | BPF_K:
770 if (!valid_alu_i(BPF_OP(code), imm)) {
771 emit_mov_i(ctx, MIPS_R_T4, imm);
772 emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
773 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
774 emit_alu_i64(ctx, dst, val, alu);
775 }
776 break;
777 /* dst = dst & src (64-bit) */
778 /* dst = dst | src (64-bit) */
779 /* dst = dst ^ src (64-bit) */
780 /* dst = dst << src (64-bit) */
781 /* dst = dst >> src (64-bit) */
782 /* dst = dst >> src (64-bit, arithmetic) */
783 /* dst = dst + src (64-bit) */
784 /* dst = dst - src (64-bit) */
785 /* dst = dst * src (64-bit) */
786 /* dst = dst / src (64-bit) */
787 /* dst = dst % src (64-bit) */
788 case BPF_ALU64 | BPF_AND | BPF_X:
789 case BPF_ALU64 | BPF_OR | BPF_X:
790 case BPF_ALU64 | BPF_XOR | BPF_X:
791 case BPF_ALU64 | BPF_LSH | BPF_X:
792 case BPF_ALU64 | BPF_RSH | BPF_X:
793 case BPF_ALU64 | BPF_ARSH | BPF_X:
794 case BPF_ALU64 | BPF_ADD | BPF_X:
795 case BPF_ALU64 | BPF_SUB | BPF_X:
796 case BPF_ALU64 | BPF_MUL | BPF_X:
797 case BPF_ALU64 | BPF_DIV | BPF_X:
798 case BPF_ALU64 | BPF_MOD | BPF_X:
799 emit_alu_r64(ctx, dst, src, BPF_OP(code));
800 break;
801 /* dst = htole(dst) */
802 /* dst = htobe(dst) */
803 case BPF_ALU | BPF_END | BPF_FROM_LE:
804 case BPF_ALU | BPF_END | BPF_FROM_BE:
805 if (BPF_SRC(code) ==
806#ifdef __BIG_ENDIAN
807 BPF_FROM_LE
808#else
809 BPF_FROM_BE
810#endif
811 )
812 emit_bswap_r64(ctx, dst, imm);
813 else
814 emit_trunc_r64(ctx, dst, imm);
815 break;
816 /* dst = imm64 */
817 case BPF_LD | BPF_IMM | BPF_DW:
818 emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32));
819 return 1;
820 /* LDX: dst = *(size *)(src + off) */
821 case BPF_LDX | BPF_MEM | BPF_W:
822 case BPF_LDX | BPF_MEM | BPF_H:
823 case BPF_LDX | BPF_MEM | BPF_B:
824 case BPF_LDX | BPF_MEM | BPF_DW:
825 emit_ldx(ctx, dst, src, off, BPF_SIZE(code));
826 break;
827 /* ST: *(size *)(dst + off) = imm */
828 case BPF_ST | BPF_MEM | BPF_W:
829 case BPF_ST | BPF_MEM | BPF_H:
830 case BPF_ST | BPF_MEM | BPF_B:
831 case BPF_ST | BPF_MEM | BPF_DW:
832 emit_mov_i(ctx, MIPS_R_T4, imm);
833 emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code));
834 break;
835 /* STX: *(size *)(dst + off) = src */
836 case BPF_STX | BPF_MEM | BPF_W:
837 case BPF_STX | BPF_MEM | BPF_H:
838 case BPF_STX | BPF_MEM | BPF_B:
839 case BPF_STX | BPF_MEM | BPF_DW:
840 emit_stx(ctx, dst, src, off, BPF_SIZE(code));
841 break;
842 /* Speculation barrier */
843 case BPF_ST | BPF_NOSPEC:
844 break;
845 /* Atomics */
846 case BPF_STX | BPF_ATOMIC | BPF_W:
847 case BPF_STX | BPF_ATOMIC | BPF_DW:
848 switch (imm) {
849 case BPF_ADD:
850 case BPF_ADD | BPF_FETCH:
851 case BPF_AND:
852 case BPF_AND | BPF_FETCH:
853 case BPF_OR:
854 case BPF_OR | BPF_FETCH:
855 case BPF_XOR:
856 case BPF_XOR | BPF_FETCH:
857 case BPF_XCHG:
858 if (BPF_SIZE(code) == BPF_DW) {
859 emit_atomic_r64(ctx, dst, src, off, imm);
860 } else if (imm & BPF_FETCH) {
861 u8 tmp = dst;
862
863 if (src == dst) { /* Don't overwrite dst */
864 emit_mov_r(ctx, MIPS_R_T4, dst);
865 tmp = MIPS_R_T4;
866 }
867 emit_sext(ctx, src, src);
868 emit_atomic_r(ctx, tmp, src, off, imm);
869 emit_zext_ver(ctx, src);
870 } else { /* 32-bit, no fetch */
871 emit_sext(ctx, MIPS_R_T4, src);
872 emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm);
873 }
874 break;
875 case BPF_CMPXCHG:
876 if (BPF_SIZE(code) == BPF_DW) {
877 emit_cmpxchg_r64(ctx, dst, src, off);
878 } else {
879 u8 tmp = res;
880
881 if (res == dst) /* Don't overwrite dst */
882 tmp = MIPS_R_T4;
883 emit_sext(ctx, tmp, res);
884 emit_sext(ctx, MIPS_R_T5, src);
885 emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off);
886 if (res == dst) /* Restore result */
887 emit_mov_r(ctx, res, MIPS_R_T4);
888 /* Result zext inserted by verifier */
889 }
890 break;
891 default:
892 goto notyet;
893 }
894 break;
895 /* PC += off if dst == src */
896 /* PC += off if dst != src */
897 /* PC += off if dst & src */
898 /* PC += off if dst > src */
899 /* PC += off if dst >= src */
900 /* PC += off if dst < src */
901 /* PC += off if dst <= src */
902 /* PC += off if dst > src (signed) */
903 /* PC += off if dst >= src (signed) */
904 /* PC += off if dst < src (signed) */
905 /* PC += off if dst <= src (signed) */
906 case BPF_JMP32 | BPF_JEQ | BPF_X:
907 case BPF_JMP32 | BPF_JNE | BPF_X:
908 case BPF_JMP32 | BPF_JSET | BPF_X:
909 case BPF_JMP32 | BPF_JGT | BPF_X:
910 case BPF_JMP32 | BPF_JGE | BPF_X:
911 case BPF_JMP32 | BPF_JLT | BPF_X:
912 case BPF_JMP32 | BPF_JLE | BPF_X:
913 case BPF_JMP32 | BPF_JSGT | BPF_X:
914 case BPF_JMP32 | BPF_JSGE | BPF_X:
915 case BPF_JMP32 | BPF_JSLT | BPF_X:
916 case BPF_JMP32 | BPF_JSLE | BPF_X:
917 if (off == 0)
918 break;
919 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
920 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
921 emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */
922 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
923 if (finish_jmp(ctx, jmp, off) < 0)
924 goto toofar;
925 break;
926 /* PC += off if dst == imm */
927 /* PC += off if dst != imm */
928 /* PC += off if dst & imm */
929 /* PC += off if dst > imm */
930 /* PC += off if dst >= imm */
931 /* PC += off if dst < imm */
932 /* PC += off if dst <= imm */
933 /* PC += off if dst > imm (signed) */
934 /* PC += off if dst >= imm (signed) */
935 /* PC += off if dst < imm (signed) */
936 /* PC += off if dst <= imm (signed) */
937 case BPF_JMP32 | BPF_JEQ | BPF_K:
938 case BPF_JMP32 | BPF_JNE | BPF_K:
939 case BPF_JMP32 | BPF_JSET | BPF_K:
940 case BPF_JMP32 | BPF_JGT | BPF_K:
941 case BPF_JMP32 | BPF_JGE | BPF_K:
942 case BPF_JMP32 | BPF_JLT | BPF_K:
943 case BPF_JMP32 | BPF_JLE | BPF_K:
944 case BPF_JMP32 | BPF_JSGT | BPF_K:
945 case BPF_JMP32 | BPF_JSGE | BPF_K:
946 case BPF_JMP32 | BPF_JSLT | BPF_K:
947 case BPF_JMP32 | BPF_JSLE | BPF_K:
948 if (off == 0)
949 break;
950 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
951 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
952 if (valid_jmp_i(jmp, imm)) {
953 emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp);
954 } else {
955 /* Move large immediate to register, sign-extended */
956 emit_mov_i(ctx, MIPS_R_T5, imm);
957 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
958 }
959 if (finish_jmp(ctx, jmp, off) < 0)
960 goto toofar;
961 break;
962 /* PC += off if dst == src */
963 /* PC += off if dst != src */
964 /* PC += off if dst & src */
965 /* PC += off if dst > src */
966 /* PC += off if dst >= src */
967 /* PC += off if dst < src */
968 /* PC += off if dst <= src */
969 /* PC += off if dst > src (signed) */
970 /* PC += off if dst >= src (signed) */
971 /* PC += off if dst < src (signed) */
972 /* PC += off if dst <= src (signed) */
973 case BPF_JMP | BPF_JEQ | BPF_X:
974 case BPF_JMP | BPF_JNE | BPF_X:
975 case BPF_JMP | BPF_JSET | BPF_X:
976 case BPF_JMP | BPF_JGT | BPF_X:
977 case BPF_JMP | BPF_JGE | BPF_X:
978 case BPF_JMP | BPF_JLT | BPF_X:
979 case BPF_JMP | BPF_JLE | BPF_X:
980 case BPF_JMP | BPF_JSGT | BPF_X:
981 case BPF_JMP | BPF_JSGE | BPF_X:
982 case BPF_JMP | BPF_JSLT | BPF_X:
983 case BPF_JMP | BPF_JSLE | BPF_X:
984 if (off == 0)
985 break;
986 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
987 emit_jmp_r(ctx, dst, src, rel, jmp);
988 if (finish_jmp(ctx, jmp, off) < 0)
989 goto toofar;
990 break;
991 /* PC += off if dst == imm */
992 /* PC += off if dst != imm */
993 /* PC += off if dst & imm */
994 /* PC += off if dst > imm */
995 /* PC += off if dst >= imm */
996 /* PC += off if dst < imm */
997 /* PC += off if dst <= imm */
998 /* PC += off if dst > imm (signed) */
999 /* PC += off if dst >= imm (signed) */
1000 /* PC += off if dst < imm (signed) */
1001 /* PC += off if dst <= imm (signed) */
1002 case BPF_JMP | BPF_JEQ | BPF_K:
1003 case BPF_JMP | BPF_JNE | BPF_K:
1004 case BPF_JMP | BPF_JSET | BPF_K:
1005 case BPF_JMP | BPF_JGT | BPF_K:
1006 case BPF_JMP | BPF_JGE | BPF_K:
1007 case BPF_JMP | BPF_JLT | BPF_K:
1008 case BPF_JMP | BPF_JLE | BPF_K:
1009 case BPF_JMP | BPF_JSGT | BPF_K:
1010 case BPF_JMP | BPF_JSGE | BPF_K:
1011 case BPF_JMP | BPF_JSLT | BPF_K:
1012 case BPF_JMP | BPF_JSLE | BPF_K:
1013 if (off == 0)
1014 break;
1015 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
1016 if (valid_jmp_i(jmp, imm)) {
1017 emit_jmp_i(ctx, dst, imm, rel, jmp);
1018 } else {
1019 /* Move large immediate to register */
1020 emit_mov_i(ctx, MIPS_R_T4, imm);
1021 emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp);
1022 }
1023 if (finish_jmp(ctx, jmp, off) < 0)
1024 goto toofar;
1025 break;
1026 /* PC += off */
1027 case BPF_JMP | BPF_JA:
1028 if (off == 0)
1029 break;
1030 if (emit_ja(ctx, off) < 0)
1031 goto toofar;
1032 break;
1033 /* Tail call */
1034 case BPF_JMP | BPF_TAIL_CALL:
1035 if (emit_tail_call(ctx) < 0)
1036 goto invalid;
1037 break;
1038 /* Function call */
1039 case BPF_JMP | BPF_CALL:
1040 if (emit_call(ctx, insn) < 0)
1041 goto invalid;
1042 break;
1043 /* Function return */
1044 case BPF_JMP | BPF_EXIT:
1045 /*
1046 * Optimization: when last instruction is EXIT
1047 * simply continue to epilogue.
1048 */
1049 if (ctx->bpf_index == ctx->program->len - 1)
1050 break;
1051 if (emit_exit(ctx) < 0)
1052 goto toofar;
1053 break;
1054
1055 default:
1056invalid:
1057 pr_err_once("unknown opcode %02x\n", code);
1058 return -EINVAL;
1059notyet:
1060 pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1061 return -EFAULT;
1062toofar:
1063 pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
1064 ctx->bpf_index, code);
1065 return -E2BIG;
1066 }
1067 return 0;
1068}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Just-In-Time compiler for eBPF bytecode on MIPS.
4 * Implementation of JIT functions for 64-bit CPUs.
5 *
6 * Copyright (c) 2021 Anyfi Networks AB.
7 * Author: Johan Almbladh <johan.almbladh@gmail.com>
8 *
9 * Based on code and ideas from
10 * Copyright (c) 2017 Cavium, Inc.
11 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
12 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
13 */
14
15#include <linux/errno.h>
16#include <linux/filter.h>
17#include <linux/bpf.h>
18#include <asm/cpu-features.h>
19#include <asm/isa-rev.h>
20#include <asm/uasm.h>
21
22#include "bpf_jit_comp.h"
23
24/* MIPS t0-t3 are not available in the n64 ABI */
25#undef MIPS_R_T0
26#undef MIPS_R_T1
27#undef MIPS_R_T2
28#undef MIPS_R_T3
29
30/* Stack is 16-byte aligned in n64 ABI */
31#define MIPS_STACK_ALIGNMENT 16
32
33/* Extra 64-bit eBPF registers used by JIT */
34#define JIT_REG_TC (MAX_BPF_JIT_REG + 0)
35#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1)
36
37/* Number of prologue bytes to skip when doing a tail call */
38#define JIT_TCALL_SKIP 4
39
40/* Callee-saved CPU registers that the JIT must preserve */
41#define JIT_CALLEE_REGS \
42 (BIT(MIPS_R_S0) | \
43 BIT(MIPS_R_S1) | \
44 BIT(MIPS_R_S2) | \
45 BIT(MIPS_R_S3) | \
46 BIT(MIPS_R_S4) | \
47 BIT(MIPS_R_S5) | \
48 BIT(MIPS_R_S6) | \
49 BIT(MIPS_R_S7) | \
50 BIT(MIPS_R_GP) | \
51 BIT(MIPS_R_FP) | \
52 BIT(MIPS_R_RA))
53
54/* Caller-saved CPU registers available for JIT use */
55#define JIT_CALLER_REGS \
56 (BIT(MIPS_R_A5) | \
57 BIT(MIPS_R_A6) | \
58 BIT(MIPS_R_A7))
59/*
60 * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers.
61 * MIPS registers t4 - t7 may be used by the JIT as temporary registers.
62 * MIPS registers t8 - t9 are reserved for single-register common functions.
63 */
64static const u8 bpf2mips64[] = {
65 /* Return value from in-kernel function, and exit value from eBPF */
66 [BPF_REG_0] = MIPS_R_V0,
67 /* Arguments from eBPF program to in-kernel function */
68 [BPF_REG_1] = MIPS_R_A0,
69 [BPF_REG_2] = MIPS_R_A1,
70 [BPF_REG_3] = MIPS_R_A2,
71 [BPF_REG_4] = MIPS_R_A3,
72 [BPF_REG_5] = MIPS_R_A4,
73 /* Callee-saved registers that in-kernel function will preserve */
74 [BPF_REG_6] = MIPS_R_S0,
75 [BPF_REG_7] = MIPS_R_S1,
76 [BPF_REG_8] = MIPS_R_S2,
77 [BPF_REG_9] = MIPS_R_S3,
78 /* Read-only frame pointer to access the eBPF stack */
79 [BPF_REG_FP] = MIPS_R_FP,
80 /* Temporary register for blinding constants */
81 [BPF_REG_AX] = MIPS_R_AT,
82 /* Tail call count register, caller-saved */
83 [JIT_REG_TC] = MIPS_R_A5,
84 /* Constant for register zero-extension */
85 [JIT_REG_ZX] = MIPS_R_V1,
86};
87
88/*
89 * MIPS 32-bit operations on 64-bit registers generate a sign-extended
90 * result. However, the eBPF ISA mandates zero-extension, so we rely on the
91 * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic
92 * operations, right shift and byte swap require properly sign-extended
93 * operands or the result is unpredictable. We emit explicit sign-extensions
94 * in those cases.
95 */
96
97/* Sign extension */
98static void emit_sext(struct jit_context *ctx, u8 dst, u8 src)
99{
100 emit(ctx, sll, dst, src, 0);
101 clobber_reg(ctx, dst);
102}
103
104/* Zero extension */
105static void emit_zext(struct jit_context *ctx, u8 dst)
106{
107 if (cpu_has_mips64r2 || cpu_has_mips64r6) {
108 emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
109 } else {
110 emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]);
111 access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */
112 }
113 clobber_reg(ctx, dst);
114}
115
116/* Zero extension, if verifier does not do it for us */
117static void emit_zext_ver(struct jit_context *ctx, u8 dst)
118{
119 if (!ctx->program->aux->verifier_zext)
120 emit_zext(ctx, dst);
121}
122
123/* dst = imm (64-bit) */
124static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64)
125{
126 if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) {
127 emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64);
128 } else if (imm64 >= 0xffffffff80000000ULL ||
129 (imm64 < 0x80000000 && imm64 > 0xffff)) {
130 emit(ctx, lui, dst, (s16)(imm64 >> 16));
131 emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff);
132 } else {
133 u8 acc = MIPS_R_ZERO;
134 int shift = 0;
135 int k;
136
137 for (k = 0; k < 4; k++) {
138 u16 half = imm64 >> (48 - 16 * k);
139
140 if (acc == dst)
141 shift += 16;
142
143 if (half) {
144 if (shift)
145 emit(ctx, dsll_safe, dst, dst, shift);
146 emit(ctx, ori, dst, acc, half);
147 acc = dst;
148 shift = 0;
149 }
150 }
151 if (shift)
152 emit(ctx, dsll_safe, dst, dst, shift);
153 }
154 clobber_reg(ctx, dst);
155}
156
157/* ALU immediate operation (64-bit) */
158static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
159{
160 switch (BPF_OP(op)) {
161 /* dst = dst | imm */
162 case BPF_OR:
163 emit(ctx, ori, dst, dst, (u16)imm);
164 break;
165 /* dst = dst ^ imm */
166 case BPF_XOR:
167 emit(ctx, xori, dst, dst, (u16)imm);
168 break;
169 /* dst = -dst */
170 case BPF_NEG:
171 emit(ctx, dsubu, dst, MIPS_R_ZERO, dst);
172 break;
173 /* dst = dst << imm */
174 case BPF_LSH:
175 emit(ctx, dsll_safe, dst, dst, imm);
176 break;
177 /* dst = dst >> imm */
178 case BPF_RSH:
179 emit(ctx, dsrl_safe, dst, dst, imm);
180 break;
181 /* dst = dst >> imm (arithmetic) */
182 case BPF_ARSH:
183 emit(ctx, dsra_safe, dst, dst, imm);
184 break;
185 /* dst = dst + imm */
186 case BPF_ADD:
187 emit(ctx, daddiu, dst, dst, imm);
188 break;
189 /* dst = dst - imm */
190 case BPF_SUB:
191 emit(ctx, daddiu, dst, dst, -imm);
192 break;
193 default:
194 /* Width-generic operations */
195 emit_alu_i(ctx, dst, imm, op);
196 }
197 clobber_reg(ctx, dst);
198}
199
200/* ALU register operation (64-bit) */
201static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
202{
203 switch (BPF_OP(op)) {
204 /* dst = dst << src */
205 case BPF_LSH:
206 emit(ctx, dsllv, dst, dst, src);
207 break;
208 /* dst = dst >> src */
209 case BPF_RSH:
210 emit(ctx, dsrlv, dst, dst, src);
211 break;
212 /* dst = dst >> src (arithmetic) */
213 case BPF_ARSH:
214 emit(ctx, dsrav, dst, dst, src);
215 break;
216 /* dst = dst + src */
217 case BPF_ADD:
218 emit(ctx, daddu, dst, dst, src);
219 break;
220 /* dst = dst - src */
221 case BPF_SUB:
222 emit(ctx, dsubu, dst, dst, src);
223 break;
224 /* dst = dst * src */
225 case BPF_MUL:
226 if (cpu_has_mips64r6) {
227 emit(ctx, dmulu, dst, dst, src);
228 } else {
229 emit(ctx, dmultu, dst, src);
230 emit(ctx, mflo, dst);
231 /* Ensure multiplication is completed */
232 if (IS_ENABLED(CONFIG_CPU_R4000_WORKAROUNDS))
233 emit(ctx, mfhi, MIPS_R_ZERO);
234 }
235 break;
236 /* dst = dst / src */
237 case BPF_DIV:
238 if (cpu_has_mips64r6) {
239 emit(ctx, ddivu_r6, dst, dst, src);
240 } else {
241 emit(ctx, ddivu, dst, src);
242 emit(ctx, mflo, dst);
243 }
244 break;
245 /* dst = dst % src */
246 case BPF_MOD:
247 if (cpu_has_mips64r6) {
248 emit(ctx, dmodu, dst, dst, src);
249 } else {
250 emit(ctx, ddivu, dst, src);
251 emit(ctx, mfhi, dst);
252 }
253 break;
254 default:
255 /* Width-generic operations */
256 emit_alu_r(ctx, dst, src, op);
257 }
258 clobber_reg(ctx, dst);
259}
260
261/* Swap sub words in a register double word */
262static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits)
263{
264 u8 tmp = MIPS_R_T9;
265
266 emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */
267 emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */
268 emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */
269 emit(ctx, and, dst, dst, mask); /* dst = dst & mask */
270 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */
271}
272
273/* Swap bytes and truncate a register double word, word or half word */
274static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width)
275{
276 switch (width) {
277 /* Swap bytes in a double word */
278 case 64:
279 if (cpu_has_mips64r2 || cpu_has_mips64r6) {
280 emit(ctx, dsbh, dst, dst);
281 emit(ctx, dshd, dst, dst);
282 } else {
283 u8 t1 = MIPS_R_T6;
284 u8 t2 = MIPS_R_T7;
285
286 emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */
287 emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */
288 emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */
289
290 emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff);
291 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
292 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
293 emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */
294
295 emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */
296 emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */
297 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
298 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
299 emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */
300 }
301 break;
302 /* Swap bytes in a half word */
303 /* Swap bytes in a word */
304 case 32:
305 case 16:
306 emit_sext(ctx, dst, dst);
307 emit_bswap_r(ctx, dst, width);
308 if (cpu_has_mips64r2 || cpu_has_mips64r6)
309 emit_zext(ctx, dst);
310 break;
311 }
312 clobber_reg(ctx, dst);
313}
314
315/* Truncate a register double word, word or half word */
316static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width)
317{
318 switch (width) {
319 case 64:
320 break;
321 /* Zero-extend a word */
322 case 32:
323 emit_zext(ctx, dst);
324 break;
325 /* Zero-extend a half word */
326 case 16:
327 emit(ctx, andi, dst, dst, 0xffff);
328 break;
329 }
330 clobber_reg(ctx, dst);
331}
332
333/* Load operation: dst = *(size*)(src + off) */
334static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
335{
336 switch (size) {
337 /* Load a byte */
338 case BPF_B:
339 emit(ctx, lbu, dst, off, src);
340 break;
341 /* Load a half word */
342 case BPF_H:
343 emit(ctx, lhu, dst, off, src);
344 break;
345 /* Load a word */
346 case BPF_W:
347 emit(ctx, lwu, dst, off, src);
348 break;
349 /* Load a double word */
350 case BPF_DW:
351 emit(ctx, ld, dst, off, src);
352 break;
353 }
354 clobber_reg(ctx, dst);
355}
356
357/* Store operation: *(size *)(dst + off) = src */
358static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
359{
360 switch (size) {
361 /* Store a byte */
362 case BPF_B:
363 emit(ctx, sb, src, off, dst);
364 break;
365 /* Store a half word */
366 case BPF_H:
367 emit(ctx, sh, src, off, dst);
368 break;
369 /* Store a word */
370 case BPF_W:
371 emit(ctx, sw, src, off, dst);
372 break;
373 /* Store a double word */
374 case BPF_DW:
375 emit(ctx, sd, src, off, dst);
376 break;
377 }
378}
379
380/* Atomic read-modify-write */
381static void emit_atomic_r64(struct jit_context *ctx,
382 u8 dst, u8 src, s16 off, u8 code)
383{
384 u8 t1 = MIPS_R_T6;
385 u8 t2 = MIPS_R_T7;
386
387 LLSC_sync(ctx);
388 emit(ctx, lld, t1, off, dst);
389 switch (code) {
390 case BPF_ADD:
391 case BPF_ADD | BPF_FETCH:
392 emit(ctx, daddu, t2, t1, src);
393 break;
394 case BPF_AND:
395 case BPF_AND | BPF_FETCH:
396 emit(ctx, and, t2, t1, src);
397 break;
398 case BPF_OR:
399 case BPF_OR | BPF_FETCH:
400 emit(ctx, or, t2, t1, src);
401 break;
402 case BPF_XOR:
403 case BPF_XOR | BPF_FETCH:
404 emit(ctx, xor, t2, t1, src);
405 break;
406 case BPF_XCHG:
407 emit(ctx, move, t2, src);
408 break;
409 }
410 emit(ctx, scd, t2, off, dst);
411 emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
412 emit(ctx, nop); /* Delay slot */
413
414 if (code & BPF_FETCH) {
415 emit(ctx, move, src, t1);
416 clobber_reg(ctx, src);
417 }
418}
419
420/* Atomic compare-and-exchange */
421static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
422{
423 u8 r0 = bpf2mips64[BPF_REG_0];
424 u8 t1 = MIPS_R_T6;
425 u8 t2 = MIPS_R_T7;
426
427 LLSC_sync(ctx);
428 emit(ctx, lld, t1, off, dst);
429 emit(ctx, bne, t1, r0, 12);
430 emit(ctx, move, t2, src); /* Delay slot */
431 emit(ctx, scd, t2, off, dst);
432 emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
433 emit(ctx, move, r0, t1); /* Delay slot */
434
435 clobber_reg(ctx, r0);
436}
437
438/* Function call */
439static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
440{
441 u8 zx = bpf2mips64[JIT_REG_ZX];
442 u8 tmp = MIPS_R_T6;
443 bool fixed;
444 u64 addr;
445
446 /* Decode the call address */
447 if (bpf_jit_get_func_addr(ctx->program, insn, false,
448 &addr, &fixed) < 0)
449 return -1;
450 if (!fixed)
451 return -1;
452
453 /* Push caller-saved registers on stack */
454 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
455
456 /* Emit function call */
457 emit_mov_i64(ctx, tmp, addr & JALR_MASK);
458 emit(ctx, jalr, MIPS_R_RA, tmp);
459 emit(ctx, nop); /* Delay slot */
460
461 /* Restore caller-saved registers */
462 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
463
464 /* Re-initialize the JIT zero-extension register if accessed */
465 if (ctx->accessed & BIT(JIT_REG_ZX)) {
466 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
467 emit(ctx, dsrl32, zx, zx, 0);
468 }
469
470 clobber_reg(ctx, MIPS_R_RA);
471 clobber_reg(ctx, MIPS_R_V0);
472 clobber_reg(ctx, MIPS_R_V1);
473 return 0;
474}
475
476/* Function tail call */
477static int emit_tail_call(struct jit_context *ctx)
478{
479 u8 ary = bpf2mips64[BPF_REG_2];
480 u8 ind = bpf2mips64[BPF_REG_3];
481 u8 tcc = bpf2mips64[JIT_REG_TC];
482 u8 tmp = MIPS_R_T6;
483 int off;
484
485 /*
486 * Tail call:
487 * eBPF R1 - function argument (context ptr), passed in a0-a1
488 * eBPF R2 - ptr to object with array of function entry points
489 * eBPF R3 - array index of function to be called
490 */
491
492 /* if (ind >= ary->map.max_entries) goto out */
493 off = offsetof(struct bpf_array, map.max_entries);
494 if (off > 0x7fff)
495 return -1;
496 emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/
497 emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */
498 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
499
500 /* if (--TCC < 0) goto out */
501 emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */
502 emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */
503 /* (next insn delay slot) */
504 /* prog = ary->ptrs[ind] */
505 off = offsetof(struct bpf_array, ptrs);
506 if (off > 0x7fff)
507 return -1;
508 emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */
509 emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */
510 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
511
512 /* if (prog == 0) goto out */
513 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
514 emit(ctx, nop); /* Delay slot */
515
516 /* func = prog->bpf_func + 8 (prologue skip offset) */
517 off = offsetof(struct bpf_prog, bpf_func);
518 if (off > 0x7fff)
519 return -1;
520 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
521 emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */
522
523 /* goto func */
524 build_epilogue(ctx, tmp);
525 access_reg(ctx, JIT_REG_TC);
526 return 0;
527}
528
529/*
530 * Stack frame layout for a JITed program (stack grows down).
531 *
532 * Higher address : Previous stack frame :
533 * +===========================+ <--- MIPS sp before call
534 * | Callee-saved registers, |
535 * | including RA and FP |
536 * +---------------------------+ <--- eBPF FP (MIPS fp)
537 * | Local eBPF variables |
538 * | allocated by program |
539 * +---------------------------+
540 * | Reserved for caller-saved |
541 * | registers |
542 * Lower address +===========================+ <--- MIPS sp
543 */
544
545/* Build program prologue to set up the stack and registers */
546void build_prologue(struct jit_context *ctx)
547{
548 u8 fp = bpf2mips64[BPF_REG_FP];
549 u8 tc = bpf2mips64[JIT_REG_TC];
550 u8 zx = bpf2mips64[JIT_REG_ZX];
551 int stack, saved, locals, reserved;
552
553 /*
554 * In the unlikely event that the TCC limit is raised to more
555 * than 16 bits, it is clamped to the maximum value allowed for
556 * the generated code (0xffff). It is better fail to compile
557 * instead of degrading gracefully.
558 */
559 BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
560
561 /*
562 * The first instruction initializes the tail call count register.
563 * On a tail call, the calling function jumps into the prologue
564 * after this instruction.
565 */
566 emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
567
568 /* === Entry-point for tail calls === */
569
570 /*
571 * If the eBPF frame pointer and tail call count registers were
572 * accessed they must be preserved. Mark them as clobbered here
573 * to save and restore them on the stack as needed.
574 */
575 if (ctx->accessed & BIT(BPF_REG_FP))
576 clobber_reg(ctx, fp);
577 if (ctx->accessed & BIT(JIT_REG_TC))
578 clobber_reg(ctx, tc);
579 if (ctx->accessed & BIT(JIT_REG_ZX))
580 clobber_reg(ctx, zx);
581
582 /* Compute the stack space needed for callee-saved registers */
583 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64);
584 saved = ALIGN(saved, MIPS_STACK_ALIGNMENT);
585
586 /* Stack space used by eBPF program local data */
587 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT);
588
589 /*
590 * If we are emitting function calls, reserve extra stack space for
591 * caller-saved registers needed by the JIT. The required space is
592 * computed automatically during resource usage discovery (pass 1).
593 */
594 reserved = ctx->stack_used;
595
596 /* Allocate the stack frame */
597 stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT);
598 if (stack)
599 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack);
600
601 /* Store callee-saved registers on stack */
602 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved);
603
604 /* Initialize the eBPF frame pointer if accessed */
605 if (ctx->accessed & BIT(BPF_REG_FP))
606 emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved);
607
608 /* Initialize the ePF JIT zero-extension register if accessed */
609 if (ctx->accessed & BIT(JIT_REG_ZX)) {
610 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
611 emit(ctx, dsrl32, zx, zx, 0);
612 }
613
614 ctx->saved_size = saved;
615 ctx->stack_size = stack;
616}
617
618/* Build the program epilogue to restore the stack and registers */
619void build_epilogue(struct jit_context *ctx, int dest_reg)
620{
621 /* Restore callee-saved registers from stack */
622 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0,
623 ctx->stack_size - ctx->saved_size);
624
625 /* Release the stack frame */
626 if (ctx->stack_size)
627 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size);
628
629 /* Jump to return address and sign-extend the 32-bit return value */
630 emit(ctx, jr, dest_reg);
631 emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */
632}
633
634/* Build one eBPF instruction */
635int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
636{
637 u8 dst = bpf2mips64[insn->dst_reg];
638 u8 src = bpf2mips64[insn->src_reg];
639 u8 res = bpf2mips64[BPF_REG_0];
640 u8 code = insn->code;
641 s16 off = insn->off;
642 s32 imm = insn->imm;
643 s32 val, rel;
644 u8 alu, jmp;
645
646 switch (code) {
647 /* ALU operations */
648 /* dst = imm */
649 case BPF_ALU | BPF_MOV | BPF_K:
650 emit_mov_i(ctx, dst, imm);
651 emit_zext_ver(ctx, dst);
652 break;
653 /* dst = src */
654 case BPF_ALU | BPF_MOV | BPF_X:
655 if (imm == 1) {
656 /* Special mov32 for zext */
657 emit_zext(ctx, dst);
658 } else {
659 emit_mov_r(ctx, dst, src);
660 emit_zext_ver(ctx, dst);
661 }
662 break;
663 /* dst = -dst */
664 case BPF_ALU | BPF_NEG:
665 emit_sext(ctx, dst, dst);
666 emit_alu_i(ctx, dst, 0, BPF_NEG);
667 emit_zext_ver(ctx, dst);
668 break;
669 /* dst = dst & imm */
670 /* dst = dst | imm */
671 /* dst = dst ^ imm */
672 /* dst = dst << imm */
673 case BPF_ALU | BPF_OR | BPF_K:
674 case BPF_ALU | BPF_AND | BPF_K:
675 case BPF_ALU | BPF_XOR | BPF_K:
676 case BPF_ALU | BPF_LSH | BPF_K:
677 if (!valid_alu_i(BPF_OP(code), imm)) {
678 emit_mov_i(ctx, MIPS_R_T4, imm);
679 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
680 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
681 emit_alu_i(ctx, dst, val, alu);
682 }
683 emit_zext_ver(ctx, dst);
684 break;
685 /* dst = dst >> imm */
686 /* dst = dst >> imm (arithmetic) */
687 /* dst = dst + imm */
688 /* dst = dst - imm */
689 /* dst = dst * imm */
690 /* dst = dst / imm */
691 /* dst = dst % imm */
692 case BPF_ALU | BPF_RSH | BPF_K:
693 case BPF_ALU | BPF_ARSH | BPF_K:
694 case BPF_ALU | BPF_ADD | BPF_K:
695 case BPF_ALU | BPF_SUB | BPF_K:
696 case BPF_ALU | BPF_MUL | BPF_K:
697 case BPF_ALU | BPF_DIV | BPF_K:
698 case BPF_ALU | BPF_MOD | BPF_K:
699 if (!valid_alu_i(BPF_OP(code), imm)) {
700 emit_sext(ctx, dst, dst);
701 emit_mov_i(ctx, MIPS_R_T4, imm);
702 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
703 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
704 emit_sext(ctx, dst, dst);
705 emit_alu_i(ctx, dst, val, alu);
706 }
707 emit_zext_ver(ctx, dst);
708 break;
709 /* dst = dst & src */
710 /* dst = dst | src */
711 /* dst = dst ^ src */
712 /* dst = dst << src */
713 case BPF_ALU | BPF_AND | BPF_X:
714 case BPF_ALU | BPF_OR | BPF_X:
715 case BPF_ALU | BPF_XOR | BPF_X:
716 case BPF_ALU | BPF_LSH | BPF_X:
717 emit_alu_r(ctx, dst, src, BPF_OP(code));
718 emit_zext_ver(ctx, dst);
719 break;
720 /* dst = dst >> src */
721 /* dst = dst >> src (arithmetic) */
722 /* dst = dst + src */
723 /* dst = dst - src */
724 /* dst = dst * src */
725 /* dst = dst / src */
726 /* dst = dst % src */
727 case BPF_ALU | BPF_RSH | BPF_X:
728 case BPF_ALU | BPF_ARSH | BPF_X:
729 case BPF_ALU | BPF_ADD | BPF_X:
730 case BPF_ALU | BPF_SUB | BPF_X:
731 case BPF_ALU | BPF_MUL | BPF_X:
732 case BPF_ALU | BPF_DIV | BPF_X:
733 case BPF_ALU | BPF_MOD | BPF_X:
734 emit_sext(ctx, dst, dst);
735 emit_sext(ctx, MIPS_R_T4, src);
736 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
737 emit_zext_ver(ctx, dst);
738 break;
739 /* dst = imm (64-bit) */
740 case BPF_ALU64 | BPF_MOV | BPF_K:
741 emit_mov_i(ctx, dst, imm);
742 break;
743 /* dst = src (64-bit) */
744 case BPF_ALU64 | BPF_MOV | BPF_X:
745 emit_mov_r(ctx, dst, src);
746 break;
747 /* dst = -dst (64-bit) */
748 case BPF_ALU64 | BPF_NEG:
749 emit_alu_i64(ctx, dst, 0, BPF_NEG);
750 break;
751 /* dst = dst & imm (64-bit) */
752 /* dst = dst | imm (64-bit) */
753 /* dst = dst ^ imm (64-bit) */
754 /* dst = dst << imm (64-bit) */
755 /* dst = dst >> imm (64-bit) */
756 /* dst = dst >> imm ((64-bit, arithmetic) */
757 /* dst = dst + imm (64-bit) */
758 /* dst = dst - imm (64-bit) */
759 /* dst = dst * imm (64-bit) */
760 /* dst = dst / imm (64-bit) */
761 /* dst = dst % imm (64-bit) */
762 case BPF_ALU64 | BPF_AND | BPF_K:
763 case BPF_ALU64 | BPF_OR | BPF_K:
764 case BPF_ALU64 | BPF_XOR | BPF_K:
765 case BPF_ALU64 | BPF_LSH | BPF_K:
766 case BPF_ALU64 | BPF_RSH | BPF_K:
767 case BPF_ALU64 | BPF_ARSH | BPF_K:
768 case BPF_ALU64 | BPF_ADD | BPF_K:
769 case BPF_ALU64 | BPF_SUB | BPF_K:
770 case BPF_ALU64 | BPF_MUL | BPF_K:
771 case BPF_ALU64 | BPF_DIV | BPF_K:
772 case BPF_ALU64 | BPF_MOD | BPF_K:
773 if (!valid_alu_i(BPF_OP(code), imm)) {
774 emit_mov_i(ctx, MIPS_R_T4, imm);
775 emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
776 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
777 emit_alu_i64(ctx, dst, val, alu);
778 }
779 break;
780 /* dst = dst & src (64-bit) */
781 /* dst = dst | src (64-bit) */
782 /* dst = dst ^ src (64-bit) */
783 /* dst = dst << src (64-bit) */
784 /* dst = dst >> src (64-bit) */
785 /* dst = dst >> src (64-bit, arithmetic) */
786 /* dst = dst + src (64-bit) */
787 /* dst = dst - src (64-bit) */
788 /* dst = dst * src (64-bit) */
789 /* dst = dst / src (64-bit) */
790 /* dst = dst % src (64-bit) */
791 case BPF_ALU64 | BPF_AND | BPF_X:
792 case BPF_ALU64 | BPF_OR | BPF_X:
793 case BPF_ALU64 | BPF_XOR | BPF_X:
794 case BPF_ALU64 | BPF_LSH | BPF_X:
795 case BPF_ALU64 | BPF_RSH | BPF_X:
796 case BPF_ALU64 | BPF_ARSH | BPF_X:
797 case BPF_ALU64 | BPF_ADD | BPF_X:
798 case BPF_ALU64 | BPF_SUB | BPF_X:
799 case BPF_ALU64 | BPF_MUL | BPF_X:
800 case BPF_ALU64 | BPF_DIV | BPF_X:
801 case BPF_ALU64 | BPF_MOD | BPF_X:
802 emit_alu_r64(ctx, dst, src, BPF_OP(code));
803 break;
804 /* dst = htole(dst) */
805 /* dst = htobe(dst) */
806 case BPF_ALU | BPF_END | BPF_FROM_LE:
807 case BPF_ALU | BPF_END | BPF_FROM_BE:
808 if (BPF_SRC(code) ==
809#ifdef __BIG_ENDIAN
810 BPF_FROM_LE
811#else
812 BPF_FROM_BE
813#endif
814 )
815 emit_bswap_r64(ctx, dst, imm);
816 else
817 emit_trunc_r64(ctx, dst, imm);
818 break;
819 /* dst = imm64 */
820 case BPF_LD | BPF_IMM | BPF_DW:
821 emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32));
822 return 1;
823 /* LDX: dst = *(size *)(src + off) */
824 case BPF_LDX | BPF_MEM | BPF_W:
825 case BPF_LDX | BPF_MEM | BPF_H:
826 case BPF_LDX | BPF_MEM | BPF_B:
827 case BPF_LDX | BPF_MEM | BPF_DW:
828 emit_ldx(ctx, dst, src, off, BPF_SIZE(code));
829 break;
830 /* ST: *(size *)(dst + off) = imm */
831 case BPF_ST | BPF_MEM | BPF_W:
832 case BPF_ST | BPF_MEM | BPF_H:
833 case BPF_ST | BPF_MEM | BPF_B:
834 case BPF_ST | BPF_MEM | BPF_DW:
835 emit_mov_i(ctx, MIPS_R_T4, imm);
836 emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code));
837 break;
838 /* STX: *(size *)(dst + off) = src */
839 case BPF_STX | BPF_MEM | BPF_W:
840 case BPF_STX | BPF_MEM | BPF_H:
841 case BPF_STX | BPF_MEM | BPF_B:
842 case BPF_STX | BPF_MEM | BPF_DW:
843 emit_stx(ctx, dst, src, off, BPF_SIZE(code));
844 break;
845 /* Speculation barrier */
846 case BPF_ST | BPF_NOSPEC:
847 break;
848 /* Atomics */
849 case BPF_STX | BPF_ATOMIC | BPF_W:
850 case BPF_STX | BPF_ATOMIC | BPF_DW:
851 switch (imm) {
852 case BPF_ADD:
853 case BPF_ADD | BPF_FETCH:
854 case BPF_AND:
855 case BPF_AND | BPF_FETCH:
856 case BPF_OR:
857 case BPF_OR | BPF_FETCH:
858 case BPF_XOR:
859 case BPF_XOR | BPF_FETCH:
860 case BPF_XCHG:
861 if (BPF_SIZE(code) == BPF_DW) {
862 emit_atomic_r64(ctx, dst, src, off, imm);
863 } else if (imm & BPF_FETCH) {
864 u8 tmp = dst;
865
866 if (src == dst) { /* Don't overwrite dst */
867 emit_mov_r(ctx, MIPS_R_T4, dst);
868 tmp = MIPS_R_T4;
869 }
870 emit_sext(ctx, src, src);
871 emit_atomic_r(ctx, tmp, src, off, imm);
872 emit_zext_ver(ctx, src);
873 } else { /* 32-bit, no fetch */
874 emit_sext(ctx, MIPS_R_T4, src);
875 emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm);
876 }
877 break;
878 case BPF_CMPXCHG:
879 if (BPF_SIZE(code) == BPF_DW) {
880 emit_cmpxchg_r64(ctx, dst, src, off);
881 } else {
882 u8 tmp = res;
883
884 if (res == dst) /* Don't overwrite dst */
885 tmp = MIPS_R_T4;
886 emit_sext(ctx, tmp, res);
887 emit_sext(ctx, MIPS_R_T5, src);
888 emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off);
889 if (res == dst) /* Restore result */
890 emit_mov_r(ctx, res, MIPS_R_T4);
891 /* Result zext inserted by verifier */
892 }
893 break;
894 default:
895 goto notyet;
896 }
897 break;
898 /* PC += off if dst == src */
899 /* PC += off if dst != src */
900 /* PC += off if dst & src */
901 /* PC += off if dst > src */
902 /* PC += off if dst >= src */
903 /* PC += off if dst < src */
904 /* PC += off if dst <= src */
905 /* PC += off if dst > src (signed) */
906 /* PC += off if dst >= src (signed) */
907 /* PC += off if dst < src (signed) */
908 /* PC += off if dst <= src (signed) */
909 case BPF_JMP32 | BPF_JEQ | BPF_X:
910 case BPF_JMP32 | BPF_JNE | BPF_X:
911 case BPF_JMP32 | BPF_JSET | BPF_X:
912 case BPF_JMP32 | BPF_JGT | BPF_X:
913 case BPF_JMP32 | BPF_JGE | BPF_X:
914 case BPF_JMP32 | BPF_JLT | BPF_X:
915 case BPF_JMP32 | BPF_JLE | BPF_X:
916 case BPF_JMP32 | BPF_JSGT | BPF_X:
917 case BPF_JMP32 | BPF_JSGE | BPF_X:
918 case BPF_JMP32 | BPF_JSLT | BPF_X:
919 case BPF_JMP32 | BPF_JSLE | BPF_X:
920 if (off == 0)
921 break;
922 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
923 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
924 emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */
925 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
926 if (finish_jmp(ctx, jmp, off) < 0)
927 goto toofar;
928 break;
929 /* PC += off if dst == imm */
930 /* PC += off if dst != imm */
931 /* PC += off if dst & imm */
932 /* PC += off if dst > imm */
933 /* PC += off if dst >= imm */
934 /* PC += off if dst < imm */
935 /* PC += off if dst <= imm */
936 /* PC += off if dst > imm (signed) */
937 /* PC += off if dst >= imm (signed) */
938 /* PC += off if dst < imm (signed) */
939 /* PC += off if dst <= imm (signed) */
940 case BPF_JMP32 | BPF_JEQ | BPF_K:
941 case BPF_JMP32 | BPF_JNE | BPF_K:
942 case BPF_JMP32 | BPF_JSET | BPF_K:
943 case BPF_JMP32 | BPF_JGT | BPF_K:
944 case BPF_JMP32 | BPF_JGE | BPF_K:
945 case BPF_JMP32 | BPF_JLT | BPF_K:
946 case BPF_JMP32 | BPF_JLE | BPF_K:
947 case BPF_JMP32 | BPF_JSGT | BPF_K:
948 case BPF_JMP32 | BPF_JSGE | BPF_K:
949 case BPF_JMP32 | BPF_JSLT | BPF_K:
950 case BPF_JMP32 | BPF_JSLE | BPF_K:
951 if (off == 0)
952 break;
953 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
954 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
955 if (valid_jmp_i(jmp, imm)) {
956 emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp);
957 } else {
958 /* Move large immediate to register, sign-extended */
959 emit_mov_i(ctx, MIPS_R_T5, imm);
960 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
961 }
962 if (finish_jmp(ctx, jmp, off) < 0)
963 goto toofar;
964 break;
965 /* PC += off if dst == src */
966 /* PC += off if dst != src */
967 /* PC += off if dst & src */
968 /* PC += off if dst > src */
969 /* PC += off if dst >= src */
970 /* PC += off if dst < src */
971 /* PC += off if dst <= src */
972 /* PC += off if dst > src (signed) */
973 /* PC += off if dst >= src (signed) */
974 /* PC += off if dst < src (signed) */
975 /* PC += off if dst <= src (signed) */
976 case BPF_JMP | BPF_JEQ | BPF_X:
977 case BPF_JMP | BPF_JNE | BPF_X:
978 case BPF_JMP | BPF_JSET | BPF_X:
979 case BPF_JMP | BPF_JGT | BPF_X:
980 case BPF_JMP | BPF_JGE | BPF_X:
981 case BPF_JMP | BPF_JLT | BPF_X:
982 case BPF_JMP | BPF_JLE | BPF_X:
983 case BPF_JMP | BPF_JSGT | BPF_X:
984 case BPF_JMP | BPF_JSGE | BPF_X:
985 case BPF_JMP | BPF_JSLT | BPF_X:
986 case BPF_JMP | BPF_JSLE | BPF_X:
987 if (off == 0)
988 break;
989 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
990 emit_jmp_r(ctx, dst, src, rel, jmp);
991 if (finish_jmp(ctx, jmp, off) < 0)
992 goto toofar;
993 break;
994 /* PC += off if dst == imm */
995 /* PC += off if dst != imm */
996 /* PC += off if dst & imm */
997 /* PC += off if dst > imm */
998 /* PC += off if dst >= imm */
999 /* PC += off if dst < imm */
1000 /* PC += off if dst <= imm */
1001 /* PC += off if dst > imm (signed) */
1002 /* PC += off if dst >= imm (signed) */
1003 /* PC += off if dst < imm (signed) */
1004 /* PC += off if dst <= imm (signed) */
1005 case BPF_JMP | BPF_JEQ | BPF_K:
1006 case BPF_JMP | BPF_JNE | BPF_K:
1007 case BPF_JMP | BPF_JSET | BPF_K:
1008 case BPF_JMP | BPF_JGT | BPF_K:
1009 case BPF_JMP | BPF_JGE | BPF_K:
1010 case BPF_JMP | BPF_JLT | BPF_K:
1011 case BPF_JMP | BPF_JLE | BPF_K:
1012 case BPF_JMP | BPF_JSGT | BPF_K:
1013 case BPF_JMP | BPF_JSGE | BPF_K:
1014 case BPF_JMP | BPF_JSLT | BPF_K:
1015 case BPF_JMP | BPF_JSLE | BPF_K:
1016 if (off == 0)
1017 break;
1018 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
1019 if (valid_jmp_i(jmp, imm)) {
1020 emit_jmp_i(ctx, dst, imm, rel, jmp);
1021 } else {
1022 /* Move large immediate to register */
1023 emit_mov_i(ctx, MIPS_R_T4, imm);
1024 emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp);
1025 }
1026 if (finish_jmp(ctx, jmp, off) < 0)
1027 goto toofar;
1028 break;
1029 /* PC += off */
1030 case BPF_JMP | BPF_JA:
1031 if (off == 0)
1032 break;
1033 if (emit_ja(ctx, off) < 0)
1034 goto toofar;
1035 break;
1036 /* Tail call */
1037 case BPF_JMP | BPF_TAIL_CALL:
1038 if (emit_tail_call(ctx) < 0)
1039 goto invalid;
1040 break;
1041 /* Function call */
1042 case BPF_JMP | BPF_CALL:
1043 if (emit_call(ctx, insn) < 0)
1044 goto invalid;
1045 break;
1046 /* Function return */
1047 case BPF_JMP | BPF_EXIT:
1048 /*
1049 * Optimization: when last instruction is EXIT
1050 * simply continue to epilogue.
1051 */
1052 if (ctx->bpf_index == ctx->program->len - 1)
1053 break;
1054 if (emit_exit(ctx) < 0)
1055 goto toofar;
1056 break;
1057
1058 default:
1059invalid:
1060 pr_err_once("unknown opcode %02x\n", code);
1061 return -EINVAL;
1062notyet:
1063 pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1064 return -EFAULT;
1065toofar:
1066 pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
1067 ctx->bpf_index, code);
1068 return -E2BIG;
1069 }
1070 return 0;
1071}