Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8#include <linux/bpf.h>
9#include <linux/filter.h>
10#include "bpf_jit.h"
11
12#define RV_REG_TCC RV_REG_A6
13#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
14
15static const int regmap[] = {
16 [BPF_REG_0] = RV_REG_A5,
17 [BPF_REG_1] = RV_REG_A0,
18 [BPF_REG_2] = RV_REG_A1,
19 [BPF_REG_3] = RV_REG_A2,
20 [BPF_REG_4] = RV_REG_A3,
21 [BPF_REG_5] = RV_REG_A4,
22 [BPF_REG_6] = RV_REG_S1,
23 [BPF_REG_7] = RV_REG_S2,
24 [BPF_REG_8] = RV_REG_S3,
25 [BPF_REG_9] = RV_REG_S4,
26 [BPF_REG_FP] = RV_REG_S5,
27 [BPF_REG_AX] = RV_REG_T0,
28};
29
30enum {
31 RV_CTX_F_SEEN_TAIL_CALL = 0,
32 RV_CTX_F_SEEN_CALL = RV_REG_RA,
33 RV_CTX_F_SEEN_S1 = RV_REG_S1,
34 RV_CTX_F_SEEN_S2 = RV_REG_S2,
35 RV_CTX_F_SEEN_S3 = RV_REG_S3,
36 RV_CTX_F_SEEN_S4 = RV_REG_S4,
37 RV_CTX_F_SEEN_S5 = RV_REG_S5,
38 RV_CTX_F_SEEN_S6 = RV_REG_S6,
39};
40
41static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
42{
43 u8 reg = regmap[bpf_reg];
44
45 switch (reg) {
46 case RV_CTX_F_SEEN_S1:
47 case RV_CTX_F_SEEN_S2:
48 case RV_CTX_F_SEEN_S3:
49 case RV_CTX_F_SEEN_S4:
50 case RV_CTX_F_SEEN_S5:
51 case RV_CTX_F_SEEN_S6:
52 __set_bit(reg, &ctx->flags);
53 }
54 return reg;
55};
56
57static bool seen_reg(int reg, struct rv_jit_context *ctx)
58{
59 switch (reg) {
60 case RV_CTX_F_SEEN_CALL:
61 case RV_CTX_F_SEEN_S1:
62 case RV_CTX_F_SEEN_S2:
63 case RV_CTX_F_SEEN_S3:
64 case RV_CTX_F_SEEN_S4:
65 case RV_CTX_F_SEEN_S5:
66 case RV_CTX_F_SEEN_S6:
67 return test_bit(reg, &ctx->flags);
68 }
69 return false;
70}
71
72static void mark_fp(struct rv_jit_context *ctx)
73{
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
75}
76
77static void mark_call(struct rv_jit_context *ctx)
78{
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
80}
81
82static bool seen_call(struct rv_jit_context *ctx)
83{
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
85}
86
87static void mark_tail_call(struct rv_jit_context *ctx)
88{
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
90}
91
92static bool seen_tail_call(struct rv_jit_context *ctx)
93{
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
95}
96
97static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
98{
99 mark_tail_call(ctx);
100
101 if (seen_call(ctx)) {
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
103 return RV_REG_S6;
104 }
105 return RV_REG_A6;
106}
107
108static bool is_32b_int(s64 val)
109{
110 return -(1L << 31) <= val && val < (1L << 31);
111}
112
113static bool in_auipc_jalr_range(s64 val)
114{
115 /*
116 * auipc+jalr can reach any signed PC-relative offset in the range
117 * [-2^31 - 2^11, 2^31 - 2^11).
118 */
119 return (-(1L << 31) - (1L << 11)) <= val &&
120 val < ((1L << 31) - (1L << 11));
121}
122
123static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
124{
125 /* Note that the immediate from the add is sign-extended,
126 * which means that we need to compensate this by adding 2^12,
127 * when the 12th bit is set. A simpler way of doing this, and
128 * getting rid of the check, is to just add 2**11 before the
129 * shift. The "Loading a 32-Bit constant" example from the
130 * "Computer Organization and Design, RISC-V edition" book by
131 * Patterson/Hennessy highlights this fact.
132 *
133 * This also means that we need to process LSB to MSB.
134 */
135 s64 upper = (val + (1 << 11)) >> 12;
136 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw,
137 * and addi are signed and RVC checks will perform signed comparisons.
138 */
139 s64 lower = ((val & 0xfff) << 52) >> 52;
140 int shift;
141
142 if (is_32b_int(val)) {
143 if (upper)
144 emit_lui(rd, upper, ctx);
145
146 if (!upper) {
147 emit_li(rd, lower, ctx);
148 return;
149 }
150
151 emit_addiw(rd, rd, lower, ctx);
152 return;
153 }
154
155 shift = __ffs(upper);
156 upper >>= shift;
157 shift += 12;
158
159 emit_imm(rd, upper, ctx);
160
161 emit_slli(rd, rd, shift, ctx);
162 if (lower)
163 emit_addi(rd, rd, lower, ctx);
164}
165
166static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
167{
168 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
169
170 if (seen_reg(RV_REG_RA, ctx)) {
171 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx);
172 store_offset -= 8;
173 }
174 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx);
175 store_offset -= 8;
176 if (seen_reg(RV_REG_S1, ctx)) {
177 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx);
178 store_offset -= 8;
179 }
180 if (seen_reg(RV_REG_S2, ctx)) {
181 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx);
182 store_offset -= 8;
183 }
184 if (seen_reg(RV_REG_S3, ctx)) {
185 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx);
186 store_offset -= 8;
187 }
188 if (seen_reg(RV_REG_S4, ctx)) {
189 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx);
190 store_offset -= 8;
191 }
192 if (seen_reg(RV_REG_S5, ctx)) {
193 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx);
194 store_offset -= 8;
195 }
196 if (seen_reg(RV_REG_S6, ctx)) {
197 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx);
198 store_offset -= 8;
199 }
200
201 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx);
202 /* Set return value. */
203 if (!is_tail_call)
204 emit_mv(RV_REG_A0, RV_REG_A5, ctx);
205 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
206 is_tail_call ? 4 : 0, /* skip TCC init */
207 ctx);
208}
209
210static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
211 struct rv_jit_context *ctx)
212{
213 switch (cond) {
214 case BPF_JEQ:
215 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
216 return;
217 case BPF_JGT:
218 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
219 return;
220 case BPF_JLT:
221 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
222 return;
223 case BPF_JGE:
224 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
225 return;
226 case BPF_JLE:
227 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
228 return;
229 case BPF_JNE:
230 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
231 return;
232 case BPF_JSGT:
233 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
234 return;
235 case BPF_JSLT:
236 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
237 return;
238 case BPF_JSGE:
239 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
240 return;
241 case BPF_JSLE:
242 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
243 }
244}
245
246static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
247 struct rv_jit_context *ctx)
248{
249 s64 upper, lower;
250
251 if (is_13b_int(rvoff)) {
252 emit_bcc(cond, rd, rs, rvoff, ctx);
253 return;
254 }
255
256 /* Adjust for jal */
257 rvoff -= 4;
258
259 /* Transform, e.g.:
260 * bne rd,rs,foo
261 * to
262 * beq rd,rs,<.L1>
263 * (auipc foo)
264 * jal(r) foo
265 * .L1
266 */
267 cond = invert_bpf_cond(cond);
268 if (is_21b_int(rvoff)) {
269 emit_bcc(cond, rd, rs, 8, ctx);
270 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
271 return;
272 }
273
274 /* 32b No need for an additional rvoff adjustment, since we
275 * get that from the auipc at PC', where PC = PC' + 4.
276 */
277 upper = (rvoff + (1 << 11)) >> 12;
278 lower = rvoff & 0xfff;
279
280 emit_bcc(cond, rd, rs, 12, ctx);
281 emit(rv_auipc(RV_REG_T1, upper), ctx);
282 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
283}
284
285static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
286{
287 emit_slli(reg, reg, 32, ctx);
288 emit_srli(reg, reg, 32, ctx);
289}
290
291static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
292{
293 int tc_ninsn, off, start_insn = ctx->ninsns;
294 u8 tcc = rv_tail_call_reg(ctx);
295
296 /* a0: &ctx
297 * a1: &array
298 * a2: index
299 *
300 * if (index >= array->map.max_entries)
301 * goto out;
302 */
303 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
304 ctx->offset[0];
305 emit_zext_32(RV_REG_A2, ctx);
306
307 off = offsetof(struct bpf_array, map.max_entries);
308 if (is_12b_check(off, insn))
309 return -1;
310 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
311 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
312 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
313
314 /* if (TCC-- < 0)
315 * goto out;
316 */
317 emit_addi(RV_REG_T1, tcc, -1, ctx);
318 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
319 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx);
320
321 /* prog = array->ptrs[index];
322 * if (!prog)
323 * goto out;
324 */
325 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx);
326 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx);
327 off = offsetof(struct bpf_array, ptrs);
328 if (is_12b_check(off, insn))
329 return -1;
330 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx);
331 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
332 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
333
334 /* goto *(prog->bpf_func + 4); */
335 off = offsetof(struct bpf_prog, bpf_func);
336 if (is_12b_check(off, insn))
337 return -1;
338 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx);
339 emit_mv(RV_REG_TCC, RV_REG_T1, ctx);
340 __build_epilogue(true, ctx);
341 return 0;
342}
343
344static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
345 struct rv_jit_context *ctx)
346{
347 u8 code = insn->code;
348
349 switch (code) {
350 case BPF_JMP | BPF_JA:
351 case BPF_JMP | BPF_CALL:
352 case BPF_JMP | BPF_EXIT:
353 case BPF_JMP | BPF_TAIL_CALL:
354 break;
355 default:
356 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
357 }
358
359 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
360 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
361 code & BPF_LDX || code & BPF_STX)
362 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
363}
364
365static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
366{
367 emit_mv(RV_REG_T2, *rd, ctx);
368 emit_zext_32(RV_REG_T2, ctx);
369 emit_mv(RV_REG_T1, *rs, ctx);
370 emit_zext_32(RV_REG_T1, ctx);
371 *rd = RV_REG_T2;
372 *rs = RV_REG_T1;
373}
374
375static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
376{
377 emit_addiw(RV_REG_T2, *rd, 0, ctx);
378 emit_addiw(RV_REG_T1, *rs, 0, ctx);
379 *rd = RV_REG_T2;
380 *rs = RV_REG_T1;
381}
382
383static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
384{
385 emit_mv(RV_REG_T2, *rd, ctx);
386 emit_zext_32(RV_REG_T2, ctx);
387 emit_zext_32(RV_REG_T1, ctx);
388 *rd = RV_REG_T2;
389}
390
391static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
392{
393 emit_addiw(RV_REG_T2, *rd, 0, ctx);
394 *rd = RV_REG_T2;
395}
396
397static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
398 struct rv_jit_context *ctx)
399{
400 s64 upper, lower;
401
402 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
403 emit(rv_jal(rd, rvoff >> 1), ctx);
404 return 0;
405 } else if (in_auipc_jalr_range(rvoff)) {
406 upper = (rvoff + (1 << 11)) >> 12;
407 lower = rvoff & 0xfff;
408 emit(rv_auipc(RV_REG_T1, upper), ctx);
409 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
410 return 0;
411 }
412
413 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff);
414 return -ERANGE;
415}
416
417static bool is_signed_bpf_cond(u8 cond)
418{
419 return cond == BPF_JSGT || cond == BPF_JSLT ||
420 cond == BPF_JSGE || cond == BPF_JSLE;
421}
422
423static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
424{
425 s64 off = 0;
426 u64 ip;
427 u8 rd;
428 int ret;
429
430 if (addr && ctx->insns) {
431 ip = (u64)(long)(ctx->insns + ctx->ninsns);
432 off = addr - ip;
433 }
434
435 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
436 if (ret)
437 return ret;
438 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
439 emit_mv(rd, RV_REG_A0, ctx);
440 return 0;
441}
442
443int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
444 bool extra_pass)
445{
446 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
447 BPF_CLASS(insn->code) == BPF_JMP;
448 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi;
449 struct bpf_prog_aux *aux = ctx->prog->aux;
450 u8 rd = -1, rs = -1, code = insn->code;
451 s16 off = insn->off;
452 s32 imm = insn->imm;
453
454 init_regs(&rd, &rs, insn, ctx);
455
456 switch (code) {
457 /* dst = src */
458 case BPF_ALU | BPF_MOV | BPF_X:
459 case BPF_ALU64 | BPF_MOV | BPF_X:
460 if (imm == 1) {
461 /* Special mov32 for zext */
462 emit_zext_32(rd, ctx);
463 break;
464 }
465 emit_mv(rd, rs, ctx);
466 if (!is64 && !aux->verifier_zext)
467 emit_zext_32(rd, ctx);
468 break;
469
470 /* dst = dst OP src */
471 case BPF_ALU | BPF_ADD | BPF_X:
472 case BPF_ALU64 | BPF_ADD | BPF_X:
473 emit_add(rd, rd, rs, ctx);
474 if (!is64 && !aux->verifier_zext)
475 emit_zext_32(rd, ctx);
476 break;
477 case BPF_ALU | BPF_SUB | BPF_X:
478 case BPF_ALU64 | BPF_SUB | BPF_X:
479 if (is64)
480 emit_sub(rd, rd, rs, ctx);
481 else
482 emit_subw(rd, rd, rs, ctx);
483
484 if (!is64 && !aux->verifier_zext)
485 emit_zext_32(rd, ctx);
486 break;
487 case BPF_ALU | BPF_AND | BPF_X:
488 case BPF_ALU64 | BPF_AND | BPF_X:
489 emit_and(rd, rd, rs, ctx);
490 if (!is64 && !aux->verifier_zext)
491 emit_zext_32(rd, ctx);
492 break;
493 case BPF_ALU | BPF_OR | BPF_X:
494 case BPF_ALU64 | BPF_OR | BPF_X:
495 emit_or(rd, rd, rs, ctx);
496 if (!is64 && !aux->verifier_zext)
497 emit_zext_32(rd, ctx);
498 break;
499 case BPF_ALU | BPF_XOR | BPF_X:
500 case BPF_ALU64 | BPF_XOR | BPF_X:
501 emit_xor(rd, rd, rs, ctx);
502 if (!is64 && !aux->verifier_zext)
503 emit_zext_32(rd, ctx);
504 break;
505 case BPF_ALU | BPF_MUL | BPF_X:
506 case BPF_ALU64 | BPF_MUL | BPF_X:
507 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
508 if (!is64 && !aux->verifier_zext)
509 emit_zext_32(rd, ctx);
510 break;
511 case BPF_ALU | BPF_DIV | BPF_X:
512 case BPF_ALU64 | BPF_DIV | BPF_X:
513 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
514 if (!is64 && !aux->verifier_zext)
515 emit_zext_32(rd, ctx);
516 break;
517 case BPF_ALU | BPF_MOD | BPF_X:
518 case BPF_ALU64 | BPF_MOD | BPF_X:
519 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
520 if (!is64 && !aux->verifier_zext)
521 emit_zext_32(rd, ctx);
522 break;
523 case BPF_ALU | BPF_LSH | BPF_X:
524 case BPF_ALU64 | BPF_LSH | BPF_X:
525 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
526 if (!is64 && !aux->verifier_zext)
527 emit_zext_32(rd, ctx);
528 break;
529 case BPF_ALU | BPF_RSH | BPF_X:
530 case BPF_ALU64 | BPF_RSH | BPF_X:
531 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
532 if (!is64 && !aux->verifier_zext)
533 emit_zext_32(rd, ctx);
534 break;
535 case BPF_ALU | BPF_ARSH | BPF_X:
536 case BPF_ALU64 | BPF_ARSH | BPF_X:
537 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
538 if (!is64 && !aux->verifier_zext)
539 emit_zext_32(rd, ctx);
540 break;
541
542 /* dst = -dst */
543 case BPF_ALU | BPF_NEG:
544 case BPF_ALU64 | BPF_NEG:
545 emit_sub(rd, RV_REG_ZERO, rd, ctx);
546 if (!is64 && !aux->verifier_zext)
547 emit_zext_32(rd, ctx);
548 break;
549
550 /* dst = BSWAP##imm(dst) */
551 case BPF_ALU | BPF_END | BPF_FROM_LE:
552 switch (imm) {
553 case 16:
554 emit_slli(rd, rd, 48, ctx);
555 emit_srli(rd, rd, 48, ctx);
556 break;
557 case 32:
558 if (!aux->verifier_zext)
559 emit_zext_32(rd, ctx);
560 break;
561 case 64:
562 /* Do nothing */
563 break;
564 }
565 break;
566
567 case BPF_ALU | BPF_END | BPF_FROM_BE:
568 emit_li(RV_REG_T2, 0, ctx);
569
570 emit_andi(RV_REG_T1, rd, 0xff, ctx);
571 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
572 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
573 emit_srli(rd, rd, 8, ctx);
574 if (imm == 16)
575 goto out_be;
576
577 emit_andi(RV_REG_T1, rd, 0xff, ctx);
578 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
579 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
580 emit_srli(rd, rd, 8, ctx);
581
582 emit_andi(RV_REG_T1, rd, 0xff, ctx);
583 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
584 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
585 emit_srli(rd, rd, 8, ctx);
586 if (imm == 32)
587 goto out_be;
588
589 emit_andi(RV_REG_T1, rd, 0xff, ctx);
590 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
591 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
592 emit_srli(rd, rd, 8, ctx);
593
594 emit_andi(RV_REG_T1, rd, 0xff, ctx);
595 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
596 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
597 emit_srli(rd, rd, 8, ctx);
598
599 emit_andi(RV_REG_T1, rd, 0xff, ctx);
600 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
601 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
602 emit_srli(rd, rd, 8, ctx);
603
604 emit_andi(RV_REG_T1, rd, 0xff, ctx);
605 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
606 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
607 emit_srli(rd, rd, 8, ctx);
608out_be:
609 emit_andi(RV_REG_T1, rd, 0xff, ctx);
610 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
611
612 emit_mv(rd, RV_REG_T2, ctx);
613 break;
614
615 /* dst = imm */
616 case BPF_ALU | BPF_MOV | BPF_K:
617 case BPF_ALU64 | BPF_MOV | BPF_K:
618 emit_imm(rd, imm, ctx);
619 if (!is64 && !aux->verifier_zext)
620 emit_zext_32(rd, ctx);
621 break;
622
623 /* dst = dst OP imm */
624 case BPF_ALU | BPF_ADD | BPF_K:
625 case BPF_ALU64 | BPF_ADD | BPF_K:
626 if (is_12b_int(imm)) {
627 emit_addi(rd, rd, imm, ctx);
628 } else {
629 emit_imm(RV_REG_T1, imm, ctx);
630 emit_add(rd, rd, RV_REG_T1, ctx);
631 }
632 if (!is64 && !aux->verifier_zext)
633 emit_zext_32(rd, ctx);
634 break;
635 case BPF_ALU | BPF_SUB | BPF_K:
636 case BPF_ALU64 | BPF_SUB | BPF_K:
637 if (is_12b_int(-imm)) {
638 emit_addi(rd, rd, -imm, ctx);
639 } else {
640 emit_imm(RV_REG_T1, imm, ctx);
641 emit_sub(rd, rd, RV_REG_T1, ctx);
642 }
643 if (!is64 && !aux->verifier_zext)
644 emit_zext_32(rd, ctx);
645 break;
646 case BPF_ALU | BPF_AND | BPF_K:
647 case BPF_ALU64 | BPF_AND | BPF_K:
648 if (is_12b_int(imm)) {
649 emit_andi(rd, rd, imm, ctx);
650 } else {
651 emit_imm(RV_REG_T1, imm, ctx);
652 emit_and(rd, rd, RV_REG_T1, ctx);
653 }
654 if (!is64 && !aux->verifier_zext)
655 emit_zext_32(rd, ctx);
656 break;
657 case BPF_ALU | BPF_OR | BPF_K:
658 case BPF_ALU64 | BPF_OR | BPF_K:
659 if (is_12b_int(imm)) {
660 emit(rv_ori(rd, rd, imm), ctx);
661 } else {
662 emit_imm(RV_REG_T1, imm, ctx);
663 emit_or(rd, rd, RV_REG_T1, ctx);
664 }
665 if (!is64 && !aux->verifier_zext)
666 emit_zext_32(rd, ctx);
667 break;
668 case BPF_ALU | BPF_XOR | BPF_K:
669 case BPF_ALU64 | BPF_XOR | BPF_K:
670 if (is_12b_int(imm)) {
671 emit(rv_xori(rd, rd, imm), ctx);
672 } else {
673 emit_imm(RV_REG_T1, imm, ctx);
674 emit_xor(rd, rd, RV_REG_T1, ctx);
675 }
676 if (!is64 && !aux->verifier_zext)
677 emit_zext_32(rd, ctx);
678 break;
679 case BPF_ALU | BPF_MUL | BPF_K:
680 case BPF_ALU64 | BPF_MUL | BPF_K:
681 emit_imm(RV_REG_T1, imm, ctx);
682 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
683 rv_mulw(rd, rd, RV_REG_T1), ctx);
684 if (!is64 && !aux->verifier_zext)
685 emit_zext_32(rd, ctx);
686 break;
687 case BPF_ALU | BPF_DIV | BPF_K:
688 case BPF_ALU64 | BPF_DIV | BPF_K:
689 emit_imm(RV_REG_T1, imm, ctx);
690 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
691 rv_divuw(rd, rd, RV_REG_T1), ctx);
692 if (!is64 && !aux->verifier_zext)
693 emit_zext_32(rd, ctx);
694 break;
695 case BPF_ALU | BPF_MOD | BPF_K:
696 case BPF_ALU64 | BPF_MOD | BPF_K:
697 emit_imm(RV_REG_T1, imm, ctx);
698 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
699 rv_remuw(rd, rd, RV_REG_T1), ctx);
700 if (!is64 && !aux->verifier_zext)
701 emit_zext_32(rd, ctx);
702 break;
703 case BPF_ALU | BPF_LSH | BPF_K:
704 case BPF_ALU64 | BPF_LSH | BPF_K:
705 emit_slli(rd, rd, imm, ctx);
706
707 if (!is64 && !aux->verifier_zext)
708 emit_zext_32(rd, ctx);
709 break;
710 case BPF_ALU | BPF_RSH | BPF_K:
711 case BPF_ALU64 | BPF_RSH | BPF_K:
712 if (is64)
713 emit_srli(rd, rd, imm, ctx);
714 else
715 emit(rv_srliw(rd, rd, imm), ctx);
716
717 if (!is64 && !aux->verifier_zext)
718 emit_zext_32(rd, ctx);
719 break;
720 case BPF_ALU | BPF_ARSH | BPF_K:
721 case BPF_ALU64 | BPF_ARSH | BPF_K:
722 if (is64)
723 emit_srai(rd, rd, imm, ctx);
724 else
725 emit(rv_sraiw(rd, rd, imm), ctx);
726
727 if (!is64 && !aux->verifier_zext)
728 emit_zext_32(rd, ctx);
729 break;
730
731 /* JUMP off */
732 case BPF_JMP | BPF_JA:
733 rvoff = rv_offset(i, off, ctx);
734 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
735 if (ret)
736 return ret;
737 break;
738
739 /* IF (dst COND src) JUMP off */
740 case BPF_JMP | BPF_JEQ | BPF_X:
741 case BPF_JMP32 | BPF_JEQ | BPF_X:
742 case BPF_JMP | BPF_JGT | BPF_X:
743 case BPF_JMP32 | BPF_JGT | BPF_X:
744 case BPF_JMP | BPF_JLT | BPF_X:
745 case BPF_JMP32 | BPF_JLT | BPF_X:
746 case BPF_JMP | BPF_JGE | BPF_X:
747 case BPF_JMP32 | BPF_JGE | BPF_X:
748 case BPF_JMP | BPF_JLE | BPF_X:
749 case BPF_JMP32 | BPF_JLE | BPF_X:
750 case BPF_JMP | BPF_JNE | BPF_X:
751 case BPF_JMP32 | BPF_JNE | BPF_X:
752 case BPF_JMP | BPF_JSGT | BPF_X:
753 case BPF_JMP32 | BPF_JSGT | BPF_X:
754 case BPF_JMP | BPF_JSLT | BPF_X:
755 case BPF_JMP32 | BPF_JSLT | BPF_X:
756 case BPF_JMP | BPF_JSGE | BPF_X:
757 case BPF_JMP32 | BPF_JSGE | BPF_X:
758 case BPF_JMP | BPF_JSLE | BPF_X:
759 case BPF_JMP32 | BPF_JSLE | BPF_X:
760 case BPF_JMP | BPF_JSET | BPF_X:
761 case BPF_JMP32 | BPF_JSET | BPF_X:
762 rvoff = rv_offset(i, off, ctx);
763 if (!is64) {
764 s = ctx->ninsns;
765 if (is_signed_bpf_cond(BPF_OP(code)))
766 emit_sext_32_rd_rs(&rd, &rs, ctx);
767 else
768 emit_zext_32_rd_rs(&rd, &rs, ctx);
769 e = ctx->ninsns;
770
771 /* Adjust for extra insns */
772 rvoff -= ninsns_rvoff(e - s);
773 }
774
775 if (BPF_OP(code) == BPF_JSET) {
776 /* Adjust for and */
777 rvoff -= 4;
778 emit_and(RV_REG_T1, rd, rs, ctx);
779 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
780 ctx);
781 } else {
782 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
783 }
784 break;
785
786 /* IF (dst COND imm) JUMP off */
787 case BPF_JMP | BPF_JEQ | BPF_K:
788 case BPF_JMP32 | BPF_JEQ | BPF_K:
789 case BPF_JMP | BPF_JGT | BPF_K:
790 case BPF_JMP32 | BPF_JGT | BPF_K:
791 case BPF_JMP | BPF_JLT | BPF_K:
792 case BPF_JMP32 | BPF_JLT | BPF_K:
793 case BPF_JMP | BPF_JGE | BPF_K:
794 case BPF_JMP32 | BPF_JGE | BPF_K:
795 case BPF_JMP | BPF_JLE | BPF_K:
796 case BPF_JMP32 | BPF_JLE | BPF_K:
797 case BPF_JMP | BPF_JNE | BPF_K:
798 case BPF_JMP32 | BPF_JNE | BPF_K:
799 case BPF_JMP | BPF_JSGT | BPF_K:
800 case BPF_JMP32 | BPF_JSGT | BPF_K:
801 case BPF_JMP | BPF_JSLT | BPF_K:
802 case BPF_JMP32 | BPF_JSLT | BPF_K:
803 case BPF_JMP | BPF_JSGE | BPF_K:
804 case BPF_JMP32 | BPF_JSGE | BPF_K:
805 case BPF_JMP | BPF_JSLE | BPF_K:
806 case BPF_JMP32 | BPF_JSLE | BPF_K:
807 rvoff = rv_offset(i, off, ctx);
808 s = ctx->ninsns;
809 if (imm) {
810 emit_imm(RV_REG_T1, imm, ctx);
811 rs = RV_REG_T1;
812 } else {
813 /* If imm is 0, simply use zero register. */
814 rs = RV_REG_ZERO;
815 }
816 if (!is64) {
817 if (is_signed_bpf_cond(BPF_OP(code)))
818 emit_sext_32_rd(&rd, ctx);
819 else
820 emit_zext_32_rd_t1(&rd, ctx);
821 }
822 e = ctx->ninsns;
823
824 /* Adjust for extra insns */
825 rvoff -= ninsns_rvoff(e - s);
826 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
827 break;
828
829 case BPF_JMP | BPF_JSET | BPF_K:
830 case BPF_JMP32 | BPF_JSET | BPF_K:
831 rvoff = rv_offset(i, off, ctx);
832 s = ctx->ninsns;
833 if (is_12b_int(imm)) {
834 emit_andi(RV_REG_T1, rd, imm, ctx);
835 } else {
836 emit_imm(RV_REG_T1, imm, ctx);
837 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx);
838 }
839 /* For jset32, we should clear the upper 32 bits of t1, but
840 * sign-extension is sufficient here and saves one instruction,
841 * as t1 is used only in comparison against zero.
842 */
843 if (!is64 && imm < 0)
844 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx);
845 e = ctx->ninsns;
846 rvoff -= ninsns_rvoff(e - s);
847 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
848 break;
849
850 /* function call */
851 case BPF_JMP | BPF_CALL:
852 {
853 bool fixed;
854 u64 addr;
855
856 mark_call(ctx);
857 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
858 &fixed);
859 if (ret < 0)
860 return ret;
861 ret = emit_call(fixed, addr, ctx);
862 if (ret)
863 return ret;
864 break;
865 }
866 /* tail call */
867 case BPF_JMP | BPF_TAIL_CALL:
868 if (emit_bpf_tail_call(i, ctx))
869 return -1;
870 break;
871
872 /* function return */
873 case BPF_JMP | BPF_EXIT:
874 if (i == ctx->prog->len - 1)
875 break;
876
877 rvoff = epilogue_offset(ctx);
878 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
879 if (ret)
880 return ret;
881 break;
882
883 /* dst = imm64 */
884 case BPF_LD | BPF_IMM | BPF_DW:
885 {
886 struct bpf_insn insn1 = insn[1];
887 u64 imm64;
888
889 imm64 = (u64)insn1.imm << 32 | (u32)imm;
890 emit_imm(rd, imm64, ctx);
891 return 1;
892 }
893
894 /* LDX: dst = *(size *)(src + off) */
895 case BPF_LDX | BPF_MEM | BPF_B:
896 if (is_12b_int(off)) {
897 emit(rv_lbu(rd, off, rs), ctx);
898 break;
899 }
900
901 emit_imm(RV_REG_T1, off, ctx);
902 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
903 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
904 if (insn_is_zext(&insn[1]))
905 return 1;
906 break;
907 case BPF_LDX | BPF_MEM | BPF_H:
908 if (is_12b_int(off)) {
909 emit(rv_lhu(rd, off, rs), ctx);
910 break;
911 }
912
913 emit_imm(RV_REG_T1, off, ctx);
914 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
915 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
916 if (insn_is_zext(&insn[1]))
917 return 1;
918 break;
919 case BPF_LDX | BPF_MEM | BPF_W:
920 if (is_12b_int(off)) {
921 emit(rv_lwu(rd, off, rs), ctx);
922 break;
923 }
924
925 emit_imm(RV_REG_T1, off, ctx);
926 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
927 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
928 if (insn_is_zext(&insn[1]))
929 return 1;
930 break;
931 case BPF_LDX | BPF_MEM | BPF_DW:
932 if (is_12b_int(off)) {
933 emit_ld(rd, off, rs, ctx);
934 break;
935 }
936
937 emit_imm(RV_REG_T1, off, ctx);
938 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
939 emit_ld(rd, 0, RV_REG_T1, ctx);
940 break;
941
942 /* ST: *(size *)(dst + off) = imm */
943 case BPF_ST | BPF_MEM | BPF_B:
944 emit_imm(RV_REG_T1, imm, ctx);
945 if (is_12b_int(off)) {
946 emit(rv_sb(rd, off, RV_REG_T1), ctx);
947 break;
948 }
949
950 emit_imm(RV_REG_T2, off, ctx);
951 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
952 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
953 break;
954
955 case BPF_ST | BPF_MEM | BPF_H:
956 emit_imm(RV_REG_T1, imm, ctx);
957 if (is_12b_int(off)) {
958 emit(rv_sh(rd, off, RV_REG_T1), ctx);
959 break;
960 }
961
962 emit_imm(RV_REG_T2, off, ctx);
963 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
964 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
965 break;
966 case BPF_ST | BPF_MEM | BPF_W:
967 emit_imm(RV_REG_T1, imm, ctx);
968 if (is_12b_int(off)) {
969 emit_sw(rd, off, RV_REG_T1, ctx);
970 break;
971 }
972
973 emit_imm(RV_REG_T2, off, ctx);
974 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
975 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx);
976 break;
977 case BPF_ST | BPF_MEM | BPF_DW:
978 emit_imm(RV_REG_T1, imm, ctx);
979 if (is_12b_int(off)) {
980 emit_sd(rd, off, RV_REG_T1, ctx);
981 break;
982 }
983
984 emit_imm(RV_REG_T2, off, ctx);
985 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
986 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx);
987 break;
988
989 /* STX: *(size *)(dst + off) = src */
990 case BPF_STX | BPF_MEM | BPF_B:
991 if (is_12b_int(off)) {
992 emit(rv_sb(rd, off, rs), ctx);
993 break;
994 }
995
996 emit_imm(RV_REG_T1, off, ctx);
997 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
998 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
999 break;
1000 case BPF_STX | BPF_MEM | BPF_H:
1001 if (is_12b_int(off)) {
1002 emit(rv_sh(rd, off, rs), ctx);
1003 break;
1004 }
1005
1006 emit_imm(RV_REG_T1, off, ctx);
1007 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1008 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
1009 break;
1010 case BPF_STX | BPF_MEM | BPF_W:
1011 if (is_12b_int(off)) {
1012 emit_sw(rd, off, rs, ctx);
1013 break;
1014 }
1015
1016 emit_imm(RV_REG_T1, off, ctx);
1017 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1018 emit_sw(RV_REG_T1, 0, rs, ctx);
1019 break;
1020 case BPF_STX | BPF_MEM | BPF_DW:
1021 if (is_12b_int(off)) {
1022 emit_sd(rd, off, rs, ctx);
1023 break;
1024 }
1025
1026 emit_imm(RV_REG_T1, off, ctx);
1027 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1028 emit_sd(RV_REG_T1, 0, rs, ctx);
1029 break;
1030 /* STX XADD: lock *(u32 *)(dst + off) += src */
1031 case BPF_STX | BPF_XADD | BPF_W:
1032 /* STX XADD: lock *(u64 *)(dst + off) += src */
1033 case BPF_STX | BPF_XADD | BPF_DW:
1034 if (off) {
1035 if (is_12b_int(off)) {
1036 emit_addi(RV_REG_T1, rd, off, ctx);
1037 } else {
1038 emit_imm(RV_REG_T1, off, ctx);
1039 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1040 }
1041
1042 rd = RV_REG_T1;
1043 }
1044
1045 emit(BPF_SIZE(code) == BPF_W ?
1046 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :
1047 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);
1048 break;
1049 default:
1050 pr_err("bpf-jit: unknown opcode %02x\n", code);
1051 return -EINVAL;
1052 }
1053
1054 return 0;
1055}
1056
1057void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1058{
1059 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1060
1061 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1062 if (bpf_stack_adjust)
1063 mark_fp(ctx);
1064
1065 if (seen_reg(RV_REG_RA, ctx))
1066 stack_adjust += 8;
1067 stack_adjust += 8; /* RV_REG_FP */
1068 if (seen_reg(RV_REG_S1, ctx))
1069 stack_adjust += 8;
1070 if (seen_reg(RV_REG_S2, ctx))
1071 stack_adjust += 8;
1072 if (seen_reg(RV_REG_S3, ctx))
1073 stack_adjust += 8;
1074 if (seen_reg(RV_REG_S4, ctx))
1075 stack_adjust += 8;
1076 if (seen_reg(RV_REG_S5, ctx))
1077 stack_adjust += 8;
1078 if (seen_reg(RV_REG_S6, ctx))
1079 stack_adjust += 8;
1080
1081 stack_adjust = round_up(stack_adjust, 16);
1082 stack_adjust += bpf_stack_adjust;
1083
1084 store_offset = stack_adjust - 8;
1085
1086 /* First instruction is always setting the tail-call-counter
1087 * (TCC) register. This instruction is skipped for tail calls.
1088 * Force using a 4-byte (non-compressed) instruction.
1089 */
1090 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1091
1092 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx);
1093
1094 if (seen_reg(RV_REG_RA, ctx)) {
1095 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx);
1096 store_offset -= 8;
1097 }
1098 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx);
1099 store_offset -= 8;
1100 if (seen_reg(RV_REG_S1, ctx)) {
1101 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx);
1102 store_offset -= 8;
1103 }
1104 if (seen_reg(RV_REG_S2, ctx)) {
1105 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx);
1106 store_offset -= 8;
1107 }
1108 if (seen_reg(RV_REG_S3, ctx)) {
1109 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx);
1110 store_offset -= 8;
1111 }
1112 if (seen_reg(RV_REG_S4, ctx)) {
1113 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx);
1114 store_offset -= 8;
1115 }
1116 if (seen_reg(RV_REG_S5, ctx)) {
1117 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx);
1118 store_offset -= 8;
1119 }
1120 if (seen_reg(RV_REG_S6, ctx)) {
1121 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx);
1122 store_offset -= 8;
1123 }
1124
1125 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx);
1126
1127 if (bpf_stack_adjust)
1128 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx);
1129
1130 /* Program contains calls and tail calls, so RV_REG_TCC need
1131 * to be saved across calls.
1132 */
1133 if (seen_tail_call(ctx) && seen_call(ctx))
1134 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx);
1135
1136 ctx->stack_size = stack_adjust;
1137}
1138
1139void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1140{
1141 __build_epilogue(false, ctx);
1142}
1143
1144void *bpf_jit_alloc_exec(unsigned long size)
1145{
1146 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
1147 BPF_JIT_REGION_END, GFP_KERNEL,
1148 PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
1149 __builtin_return_address(0));
1150}
1151
1152void bpf_jit_free_exec(void *addr)
1153{
1154 return vfree(addr);
1155}
1// SPDX-License-Identifier: GPL-2.0
2/* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8#include <linux/bitfield.h>
9#include <linux/bpf.h>
10#include <linux/filter.h>
11#include "bpf_jit.h"
12
13#define RV_REG_TCC RV_REG_A6
14#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
15
16static const int regmap[] = {
17 [BPF_REG_0] = RV_REG_A5,
18 [BPF_REG_1] = RV_REG_A0,
19 [BPF_REG_2] = RV_REG_A1,
20 [BPF_REG_3] = RV_REG_A2,
21 [BPF_REG_4] = RV_REG_A3,
22 [BPF_REG_5] = RV_REG_A4,
23 [BPF_REG_6] = RV_REG_S1,
24 [BPF_REG_7] = RV_REG_S2,
25 [BPF_REG_8] = RV_REG_S3,
26 [BPF_REG_9] = RV_REG_S4,
27 [BPF_REG_FP] = RV_REG_S5,
28 [BPF_REG_AX] = RV_REG_T0,
29};
30
31static const int pt_regmap[] = {
32 [RV_REG_A0] = offsetof(struct pt_regs, a0),
33 [RV_REG_A1] = offsetof(struct pt_regs, a1),
34 [RV_REG_A2] = offsetof(struct pt_regs, a2),
35 [RV_REG_A3] = offsetof(struct pt_regs, a3),
36 [RV_REG_A4] = offsetof(struct pt_regs, a4),
37 [RV_REG_A5] = offsetof(struct pt_regs, a5),
38 [RV_REG_S1] = offsetof(struct pt_regs, s1),
39 [RV_REG_S2] = offsetof(struct pt_regs, s2),
40 [RV_REG_S3] = offsetof(struct pt_regs, s3),
41 [RV_REG_S4] = offsetof(struct pt_regs, s4),
42 [RV_REG_S5] = offsetof(struct pt_regs, s5),
43 [RV_REG_T0] = offsetof(struct pt_regs, t0),
44};
45
46enum {
47 RV_CTX_F_SEEN_TAIL_CALL = 0,
48 RV_CTX_F_SEEN_CALL = RV_REG_RA,
49 RV_CTX_F_SEEN_S1 = RV_REG_S1,
50 RV_CTX_F_SEEN_S2 = RV_REG_S2,
51 RV_CTX_F_SEEN_S3 = RV_REG_S3,
52 RV_CTX_F_SEEN_S4 = RV_REG_S4,
53 RV_CTX_F_SEEN_S5 = RV_REG_S5,
54 RV_CTX_F_SEEN_S6 = RV_REG_S6,
55};
56
57static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
58{
59 u8 reg = regmap[bpf_reg];
60
61 switch (reg) {
62 case RV_CTX_F_SEEN_S1:
63 case RV_CTX_F_SEEN_S2:
64 case RV_CTX_F_SEEN_S3:
65 case RV_CTX_F_SEEN_S4:
66 case RV_CTX_F_SEEN_S5:
67 case RV_CTX_F_SEEN_S6:
68 __set_bit(reg, &ctx->flags);
69 }
70 return reg;
71};
72
73static bool seen_reg(int reg, struct rv_jit_context *ctx)
74{
75 switch (reg) {
76 case RV_CTX_F_SEEN_CALL:
77 case RV_CTX_F_SEEN_S1:
78 case RV_CTX_F_SEEN_S2:
79 case RV_CTX_F_SEEN_S3:
80 case RV_CTX_F_SEEN_S4:
81 case RV_CTX_F_SEEN_S5:
82 case RV_CTX_F_SEEN_S6:
83 return test_bit(reg, &ctx->flags);
84 }
85 return false;
86}
87
88static void mark_fp(struct rv_jit_context *ctx)
89{
90 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
91}
92
93static void mark_call(struct rv_jit_context *ctx)
94{
95 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
96}
97
98static bool seen_call(struct rv_jit_context *ctx)
99{
100 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
101}
102
103static void mark_tail_call(struct rv_jit_context *ctx)
104{
105 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
106}
107
108static bool seen_tail_call(struct rv_jit_context *ctx)
109{
110 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
111}
112
113static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
114{
115 mark_tail_call(ctx);
116
117 if (seen_call(ctx)) {
118 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
119 return RV_REG_S6;
120 }
121 return RV_REG_A6;
122}
123
124static bool is_32b_int(s64 val)
125{
126 return -(1L << 31) <= val && val < (1L << 31);
127}
128
129static bool in_auipc_jalr_range(s64 val)
130{
131 /*
132 * auipc+jalr can reach any signed PC-relative offset in the range
133 * [-2^31 - 2^11, 2^31 - 2^11).
134 */
135 return (-(1L << 31) - (1L << 11)) <= val &&
136 val < ((1L << 31) - (1L << 11));
137}
138
139/* Emit fixed-length instructions for address */
140static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx)
141{
142 u64 ip = (u64)(ctx->insns + ctx->ninsns);
143 s64 off = addr - ip;
144 s64 upper = (off + (1 << 11)) >> 12;
145 s64 lower = off & 0xfff;
146
147 if (extra_pass && !in_auipc_jalr_range(off)) {
148 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off);
149 return -ERANGE;
150 }
151
152 emit(rv_auipc(rd, upper), ctx);
153 emit(rv_addi(rd, rd, lower), ctx);
154 return 0;
155}
156
157/* Emit variable-length instructions for 32-bit and 64-bit imm */
158static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
159{
160 /* Note that the immediate from the add is sign-extended,
161 * which means that we need to compensate this by adding 2^12,
162 * when the 12th bit is set. A simpler way of doing this, and
163 * getting rid of the check, is to just add 2**11 before the
164 * shift. The "Loading a 32-Bit constant" example from the
165 * "Computer Organization and Design, RISC-V edition" book by
166 * Patterson/Hennessy highlights this fact.
167 *
168 * This also means that we need to process LSB to MSB.
169 */
170 s64 upper = (val + (1 << 11)) >> 12;
171 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw,
172 * and addi are signed and RVC checks will perform signed comparisons.
173 */
174 s64 lower = ((val & 0xfff) << 52) >> 52;
175 int shift;
176
177 if (is_32b_int(val)) {
178 if (upper)
179 emit_lui(rd, upper, ctx);
180
181 if (!upper) {
182 emit_li(rd, lower, ctx);
183 return;
184 }
185
186 emit_addiw(rd, rd, lower, ctx);
187 return;
188 }
189
190 shift = __ffs(upper);
191 upper >>= shift;
192 shift += 12;
193
194 emit_imm(rd, upper, ctx);
195
196 emit_slli(rd, rd, shift, ctx);
197 if (lower)
198 emit_addi(rd, rd, lower, ctx);
199}
200
201static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
202{
203 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
204
205 if (seen_reg(RV_REG_RA, ctx)) {
206 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx);
207 store_offset -= 8;
208 }
209 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx);
210 store_offset -= 8;
211 if (seen_reg(RV_REG_S1, ctx)) {
212 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx);
213 store_offset -= 8;
214 }
215 if (seen_reg(RV_REG_S2, ctx)) {
216 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx);
217 store_offset -= 8;
218 }
219 if (seen_reg(RV_REG_S3, ctx)) {
220 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx);
221 store_offset -= 8;
222 }
223 if (seen_reg(RV_REG_S4, ctx)) {
224 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx);
225 store_offset -= 8;
226 }
227 if (seen_reg(RV_REG_S5, ctx)) {
228 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx);
229 store_offset -= 8;
230 }
231 if (seen_reg(RV_REG_S6, ctx)) {
232 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx);
233 store_offset -= 8;
234 }
235
236 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx);
237 /* Set return value. */
238 if (!is_tail_call)
239 emit_mv(RV_REG_A0, RV_REG_A5, ctx);
240 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
241 is_tail_call ? 4 : 0, /* skip TCC init */
242 ctx);
243}
244
245static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
246 struct rv_jit_context *ctx)
247{
248 switch (cond) {
249 case BPF_JEQ:
250 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
251 return;
252 case BPF_JGT:
253 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
254 return;
255 case BPF_JLT:
256 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
257 return;
258 case BPF_JGE:
259 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
260 return;
261 case BPF_JLE:
262 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
263 return;
264 case BPF_JNE:
265 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
266 return;
267 case BPF_JSGT:
268 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
269 return;
270 case BPF_JSLT:
271 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
272 return;
273 case BPF_JSGE:
274 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
275 return;
276 case BPF_JSLE:
277 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
278 }
279}
280
281static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
282 struct rv_jit_context *ctx)
283{
284 s64 upper, lower;
285
286 if (is_13b_int(rvoff)) {
287 emit_bcc(cond, rd, rs, rvoff, ctx);
288 return;
289 }
290
291 /* Adjust for jal */
292 rvoff -= 4;
293
294 /* Transform, e.g.:
295 * bne rd,rs,foo
296 * to
297 * beq rd,rs,<.L1>
298 * (auipc foo)
299 * jal(r) foo
300 * .L1
301 */
302 cond = invert_bpf_cond(cond);
303 if (is_21b_int(rvoff)) {
304 emit_bcc(cond, rd, rs, 8, ctx);
305 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
306 return;
307 }
308
309 /* 32b No need for an additional rvoff adjustment, since we
310 * get that from the auipc at PC', where PC = PC' + 4.
311 */
312 upper = (rvoff + (1 << 11)) >> 12;
313 lower = rvoff & 0xfff;
314
315 emit_bcc(cond, rd, rs, 12, ctx);
316 emit(rv_auipc(RV_REG_T1, upper), ctx);
317 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
318}
319
320static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
321{
322 emit_slli(reg, reg, 32, ctx);
323 emit_srli(reg, reg, 32, ctx);
324}
325
326static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
327{
328 int tc_ninsn, off, start_insn = ctx->ninsns;
329 u8 tcc = rv_tail_call_reg(ctx);
330
331 /* a0: &ctx
332 * a1: &array
333 * a2: index
334 *
335 * if (index >= array->map.max_entries)
336 * goto out;
337 */
338 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
339 ctx->offset[0];
340 emit_zext_32(RV_REG_A2, ctx);
341
342 off = offsetof(struct bpf_array, map.max_entries);
343 if (is_12b_check(off, insn))
344 return -1;
345 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
346 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
347 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
348
349 /* if (--TCC < 0)
350 * goto out;
351 */
352 emit_addi(RV_REG_TCC, tcc, -1, ctx);
353 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
354 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);
355
356 /* prog = array->ptrs[index];
357 * if (!prog)
358 * goto out;
359 */
360 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx);
361 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx);
362 off = offsetof(struct bpf_array, ptrs);
363 if (is_12b_check(off, insn))
364 return -1;
365 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx);
366 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
367 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
368
369 /* goto *(prog->bpf_func + 4); */
370 off = offsetof(struct bpf_prog, bpf_func);
371 if (is_12b_check(off, insn))
372 return -1;
373 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx);
374 __build_epilogue(true, ctx);
375 return 0;
376}
377
378static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
379 struct rv_jit_context *ctx)
380{
381 u8 code = insn->code;
382
383 switch (code) {
384 case BPF_JMP | BPF_JA:
385 case BPF_JMP | BPF_CALL:
386 case BPF_JMP | BPF_EXIT:
387 case BPF_JMP | BPF_TAIL_CALL:
388 break;
389 default:
390 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
391 }
392
393 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
394 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
395 code & BPF_LDX || code & BPF_STX)
396 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
397}
398
399static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
400{
401 emit_mv(RV_REG_T2, *rd, ctx);
402 emit_zext_32(RV_REG_T2, ctx);
403 emit_mv(RV_REG_T1, *rs, ctx);
404 emit_zext_32(RV_REG_T1, ctx);
405 *rd = RV_REG_T2;
406 *rs = RV_REG_T1;
407}
408
409static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
410{
411 emit_addiw(RV_REG_T2, *rd, 0, ctx);
412 emit_addiw(RV_REG_T1, *rs, 0, ctx);
413 *rd = RV_REG_T2;
414 *rs = RV_REG_T1;
415}
416
417static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
418{
419 emit_mv(RV_REG_T2, *rd, ctx);
420 emit_zext_32(RV_REG_T2, ctx);
421 emit_zext_32(RV_REG_T1, ctx);
422 *rd = RV_REG_T2;
423}
424
425static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
426{
427 emit_addiw(RV_REG_T2, *rd, 0, ctx);
428 *rd = RV_REG_T2;
429}
430
431static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
432 struct rv_jit_context *ctx)
433{
434 s64 upper, lower;
435
436 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
437 emit(rv_jal(rd, rvoff >> 1), ctx);
438 return 0;
439 } else if (in_auipc_jalr_range(rvoff)) {
440 upper = (rvoff + (1 << 11)) >> 12;
441 lower = rvoff & 0xfff;
442 emit(rv_auipc(RV_REG_T1, upper), ctx);
443 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
444 return 0;
445 }
446
447 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff);
448 return -ERANGE;
449}
450
451static bool is_signed_bpf_cond(u8 cond)
452{
453 return cond == BPF_JSGT || cond == BPF_JSLT ||
454 cond == BPF_JSGE || cond == BPF_JSLE;
455}
456
457static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
458{
459 s64 off = 0;
460 u64 ip;
461 u8 rd;
462 int ret;
463
464 if (addr && ctx->insns) {
465 ip = (u64)(long)(ctx->insns + ctx->ninsns);
466 off = addr - ip;
467 }
468
469 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
470 if (ret)
471 return ret;
472 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
473 emit_mv(rd, RV_REG_A0, ctx);
474 return 0;
475}
476
477static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
478 struct rv_jit_context *ctx)
479{
480 u8 r0;
481 int jmp_offset;
482
483 if (off) {
484 if (is_12b_int(off)) {
485 emit_addi(RV_REG_T1, rd, off, ctx);
486 } else {
487 emit_imm(RV_REG_T1, off, ctx);
488 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
489 }
490 rd = RV_REG_T1;
491 }
492
493 switch (imm) {
494 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */
495 case BPF_ADD:
496 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) :
497 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
498 break;
499 case BPF_AND:
500 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) :
501 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
502 break;
503 case BPF_OR:
504 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) :
505 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
506 break;
507 case BPF_XOR:
508 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) :
509 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
510 break;
511 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
512 case BPF_ADD | BPF_FETCH:
513 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) :
514 rv_amoadd_w(rs, rs, rd, 0, 0), ctx);
515 if (!is64)
516 emit_zext_32(rs, ctx);
517 break;
518 case BPF_AND | BPF_FETCH:
519 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) :
520 rv_amoand_w(rs, rs, rd, 0, 0), ctx);
521 if (!is64)
522 emit_zext_32(rs, ctx);
523 break;
524 case BPF_OR | BPF_FETCH:
525 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) :
526 rv_amoor_w(rs, rs, rd, 0, 0), ctx);
527 if (!is64)
528 emit_zext_32(rs, ctx);
529 break;
530 case BPF_XOR | BPF_FETCH:
531 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) :
532 rv_amoxor_w(rs, rs, rd, 0, 0), ctx);
533 if (!is64)
534 emit_zext_32(rs, ctx);
535 break;
536 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
537 case BPF_XCHG:
538 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) :
539 rv_amoswap_w(rs, rs, rd, 0, 0), ctx);
540 if (!is64)
541 emit_zext_32(rs, ctx);
542 break;
543 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
544 case BPF_CMPXCHG:
545 r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
546 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) :
547 rv_addiw(RV_REG_T2, r0, 0), ctx);
548 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
549 rv_lr_w(r0, 0, rd, 0, 0), ctx);
550 jmp_offset = ninsns_rvoff(8);
551 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
552 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) :
553 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx);
554 jmp_offset = ninsns_rvoff(-6);
555 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
556 emit(rv_fence(0x3, 0x3), ctx);
557 break;
558 }
559}
560
561#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
562#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
563
564bool ex_handler_bpf(const struct exception_table_entry *ex,
565 struct pt_regs *regs)
566{
567 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
568 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
569
570 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0;
571 regs->epc = (unsigned long)&ex->fixup - offset;
572
573 return true;
574}
575
576/* For accesses to BTF pointers, add an entry to the exception table */
577static int add_exception_handler(const struct bpf_insn *insn,
578 struct rv_jit_context *ctx,
579 int dst_reg, int insn_len)
580{
581 struct exception_table_entry *ex;
582 unsigned long pc;
583 off_t offset;
584
585 if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)
586 return 0;
587
588 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
589 return -EINVAL;
590
591 if (WARN_ON_ONCE(insn_len > ctx->ninsns))
592 return -EINVAL;
593
594 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1))
595 return -EINVAL;
596
597 ex = &ctx->prog->aux->extable[ctx->nexentries];
598 pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len];
599
600 offset = pc - (long)&ex->insn;
601 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
602 return -ERANGE;
603 ex->insn = offset;
604
605 /*
606 * Since the extable follows the program, the fixup offset is always
607 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
608 * to keep things simple, and put the destination register in the upper
609 * bits. We don't need to worry about buildtime or runtime sort
610 * modifying the upper bits because the table is already sorted, and
611 * isn't part of the main exception table.
612 */
613 offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
614 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
615 return -ERANGE;
616
617 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
618 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
619 ex->type = EX_TYPE_BPF;
620
621 ctx->nexentries++;
622 return 0;
623}
624
625int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
626 bool extra_pass)
627{
628 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
629 BPF_CLASS(insn->code) == BPF_JMP;
630 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi;
631 struct bpf_prog_aux *aux = ctx->prog->aux;
632 u8 rd = -1, rs = -1, code = insn->code;
633 s16 off = insn->off;
634 s32 imm = insn->imm;
635
636 init_regs(&rd, &rs, insn, ctx);
637
638 switch (code) {
639 /* dst = src */
640 case BPF_ALU | BPF_MOV | BPF_X:
641 case BPF_ALU64 | BPF_MOV | BPF_X:
642 if (imm == 1) {
643 /* Special mov32 for zext */
644 emit_zext_32(rd, ctx);
645 break;
646 }
647 emit_mv(rd, rs, ctx);
648 if (!is64 && !aux->verifier_zext)
649 emit_zext_32(rd, ctx);
650 break;
651
652 /* dst = dst OP src */
653 case BPF_ALU | BPF_ADD | BPF_X:
654 case BPF_ALU64 | BPF_ADD | BPF_X:
655 emit_add(rd, rd, rs, ctx);
656 if (!is64 && !aux->verifier_zext)
657 emit_zext_32(rd, ctx);
658 break;
659 case BPF_ALU | BPF_SUB | BPF_X:
660 case BPF_ALU64 | BPF_SUB | BPF_X:
661 if (is64)
662 emit_sub(rd, rd, rs, ctx);
663 else
664 emit_subw(rd, rd, rs, ctx);
665
666 if (!is64 && !aux->verifier_zext)
667 emit_zext_32(rd, ctx);
668 break;
669 case BPF_ALU | BPF_AND | BPF_X:
670 case BPF_ALU64 | BPF_AND | BPF_X:
671 emit_and(rd, rd, rs, ctx);
672 if (!is64 && !aux->verifier_zext)
673 emit_zext_32(rd, ctx);
674 break;
675 case BPF_ALU | BPF_OR | BPF_X:
676 case BPF_ALU64 | BPF_OR | BPF_X:
677 emit_or(rd, rd, rs, ctx);
678 if (!is64 && !aux->verifier_zext)
679 emit_zext_32(rd, ctx);
680 break;
681 case BPF_ALU | BPF_XOR | BPF_X:
682 case BPF_ALU64 | BPF_XOR | BPF_X:
683 emit_xor(rd, rd, rs, ctx);
684 if (!is64 && !aux->verifier_zext)
685 emit_zext_32(rd, ctx);
686 break;
687 case BPF_ALU | BPF_MUL | BPF_X:
688 case BPF_ALU64 | BPF_MUL | BPF_X:
689 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
690 if (!is64 && !aux->verifier_zext)
691 emit_zext_32(rd, ctx);
692 break;
693 case BPF_ALU | BPF_DIV | BPF_X:
694 case BPF_ALU64 | BPF_DIV | BPF_X:
695 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
696 if (!is64 && !aux->verifier_zext)
697 emit_zext_32(rd, ctx);
698 break;
699 case BPF_ALU | BPF_MOD | BPF_X:
700 case BPF_ALU64 | BPF_MOD | BPF_X:
701 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
702 if (!is64 && !aux->verifier_zext)
703 emit_zext_32(rd, ctx);
704 break;
705 case BPF_ALU | BPF_LSH | BPF_X:
706 case BPF_ALU64 | BPF_LSH | BPF_X:
707 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
708 if (!is64 && !aux->verifier_zext)
709 emit_zext_32(rd, ctx);
710 break;
711 case BPF_ALU | BPF_RSH | BPF_X:
712 case BPF_ALU64 | BPF_RSH | BPF_X:
713 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
714 if (!is64 && !aux->verifier_zext)
715 emit_zext_32(rd, ctx);
716 break;
717 case BPF_ALU | BPF_ARSH | BPF_X:
718 case BPF_ALU64 | BPF_ARSH | BPF_X:
719 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
720 if (!is64 && !aux->verifier_zext)
721 emit_zext_32(rd, ctx);
722 break;
723
724 /* dst = -dst */
725 case BPF_ALU | BPF_NEG:
726 case BPF_ALU64 | BPF_NEG:
727 emit_sub(rd, RV_REG_ZERO, rd, ctx);
728 if (!is64 && !aux->verifier_zext)
729 emit_zext_32(rd, ctx);
730 break;
731
732 /* dst = BSWAP##imm(dst) */
733 case BPF_ALU | BPF_END | BPF_FROM_LE:
734 switch (imm) {
735 case 16:
736 emit_slli(rd, rd, 48, ctx);
737 emit_srli(rd, rd, 48, ctx);
738 break;
739 case 32:
740 if (!aux->verifier_zext)
741 emit_zext_32(rd, ctx);
742 break;
743 case 64:
744 /* Do nothing */
745 break;
746 }
747 break;
748
749 case BPF_ALU | BPF_END | BPF_FROM_BE:
750 emit_li(RV_REG_T2, 0, ctx);
751
752 emit_andi(RV_REG_T1, rd, 0xff, ctx);
753 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
754 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
755 emit_srli(rd, rd, 8, ctx);
756 if (imm == 16)
757 goto out_be;
758
759 emit_andi(RV_REG_T1, rd, 0xff, ctx);
760 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
761 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
762 emit_srli(rd, rd, 8, ctx);
763
764 emit_andi(RV_REG_T1, rd, 0xff, ctx);
765 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
766 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
767 emit_srli(rd, rd, 8, ctx);
768 if (imm == 32)
769 goto out_be;
770
771 emit_andi(RV_REG_T1, rd, 0xff, ctx);
772 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
773 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
774 emit_srli(rd, rd, 8, ctx);
775
776 emit_andi(RV_REG_T1, rd, 0xff, ctx);
777 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
778 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
779 emit_srli(rd, rd, 8, ctx);
780
781 emit_andi(RV_REG_T1, rd, 0xff, ctx);
782 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
783 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
784 emit_srli(rd, rd, 8, ctx);
785
786 emit_andi(RV_REG_T1, rd, 0xff, ctx);
787 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
788 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
789 emit_srli(rd, rd, 8, ctx);
790out_be:
791 emit_andi(RV_REG_T1, rd, 0xff, ctx);
792 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
793
794 emit_mv(rd, RV_REG_T2, ctx);
795 break;
796
797 /* dst = imm */
798 case BPF_ALU | BPF_MOV | BPF_K:
799 case BPF_ALU64 | BPF_MOV | BPF_K:
800 emit_imm(rd, imm, ctx);
801 if (!is64 && !aux->verifier_zext)
802 emit_zext_32(rd, ctx);
803 break;
804
805 /* dst = dst OP imm */
806 case BPF_ALU | BPF_ADD | BPF_K:
807 case BPF_ALU64 | BPF_ADD | BPF_K:
808 if (is_12b_int(imm)) {
809 emit_addi(rd, rd, imm, ctx);
810 } else {
811 emit_imm(RV_REG_T1, imm, ctx);
812 emit_add(rd, rd, RV_REG_T1, ctx);
813 }
814 if (!is64 && !aux->verifier_zext)
815 emit_zext_32(rd, ctx);
816 break;
817 case BPF_ALU | BPF_SUB | BPF_K:
818 case BPF_ALU64 | BPF_SUB | BPF_K:
819 if (is_12b_int(-imm)) {
820 emit_addi(rd, rd, -imm, ctx);
821 } else {
822 emit_imm(RV_REG_T1, imm, ctx);
823 emit_sub(rd, rd, RV_REG_T1, ctx);
824 }
825 if (!is64 && !aux->verifier_zext)
826 emit_zext_32(rd, ctx);
827 break;
828 case BPF_ALU | BPF_AND | BPF_K:
829 case BPF_ALU64 | BPF_AND | BPF_K:
830 if (is_12b_int(imm)) {
831 emit_andi(rd, rd, imm, ctx);
832 } else {
833 emit_imm(RV_REG_T1, imm, ctx);
834 emit_and(rd, rd, RV_REG_T1, ctx);
835 }
836 if (!is64 && !aux->verifier_zext)
837 emit_zext_32(rd, ctx);
838 break;
839 case BPF_ALU | BPF_OR | BPF_K:
840 case BPF_ALU64 | BPF_OR | BPF_K:
841 if (is_12b_int(imm)) {
842 emit(rv_ori(rd, rd, imm), ctx);
843 } else {
844 emit_imm(RV_REG_T1, imm, ctx);
845 emit_or(rd, rd, RV_REG_T1, ctx);
846 }
847 if (!is64 && !aux->verifier_zext)
848 emit_zext_32(rd, ctx);
849 break;
850 case BPF_ALU | BPF_XOR | BPF_K:
851 case BPF_ALU64 | BPF_XOR | BPF_K:
852 if (is_12b_int(imm)) {
853 emit(rv_xori(rd, rd, imm), ctx);
854 } else {
855 emit_imm(RV_REG_T1, imm, ctx);
856 emit_xor(rd, rd, RV_REG_T1, ctx);
857 }
858 if (!is64 && !aux->verifier_zext)
859 emit_zext_32(rd, ctx);
860 break;
861 case BPF_ALU | BPF_MUL | BPF_K:
862 case BPF_ALU64 | BPF_MUL | BPF_K:
863 emit_imm(RV_REG_T1, imm, ctx);
864 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
865 rv_mulw(rd, rd, RV_REG_T1), ctx);
866 if (!is64 && !aux->verifier_zext)
867 emit_zext_32(rd, ctx);
868 break;
869 case BPF_ALU | BPF_DIV | BPF_K:
870 case BPF_ALU64 | BPF_DIV | BPF_K:
871 emit_imm(RV_REG_T1, imm, ctx);
872 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
873 rv_divuw(rd, rd, RV_REG_T1), ctx);
874 if (!is64 && !aux->verifier_zext)
875 emit_zext_32(rd, ctx);
876 break;
877 case BPF_ALU | BPF_MOD | BPF_K:
878 case BPF_ALU64 | BPF_MOD | BPF_K:
879 emit_imm(RV_REG_T1, imm, ctx);
880 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
881 rv_remuw(rd, rd, RV_REG_T1), ctx);
882 if (!is64 && !aux->verifier_zext)
883 emit_zext_32(rd, ctx);
884 break;
885 case BPF_ALU | BPF_LSH | BPF_K:
886 case BPF_ALU64 | BPF_LSH | BPF_K:
887 emit_slli(rd, rd, imm, ctx);
888
889 if (!is64 && !aux->verifier_zext)
890 emit_zext_32(rd, ctx);
891 break;
892 case BPF_ALU | BPF_RSH | BPF_K:
893 case BPF_ALU64 | BPF_RSH | BPF_K:
894 if (is64)
895 emit_srli(rd, rd, imm, ctx);
896 else
897 emit(rv_srliw(rd, rd, imm), ctx);
898
899 if (!is64 && !aux->verifier_zext)
900 emit_zext_32(rd, ctx);
901 break;
902 case BPF_ALU | BPF_ARSH | BPF_K:
903 case BPF_ALU64 | BPF_ARSH | BPF_K:
904 if (is64)
905 emit_srai(rd, rd, imm, ctx);
906 else
907 emit(rv_sraiw(rd, rd, imm), ctx);
908
909 if (!is64 && !aux->verifier_zext)
910 emit_zext_32(rd, ctx);
911 break;
912
913 /* JUMP off */
914 case BPF_JMP | BPF_JA:
915 rvoff = rv_offset(i, off, ctx);
916 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
917 if (ret)
918 return ret;
919 break;
920
921 /* IF (dst COND src) JUMP off */
922 case BPF_JMP | BPF_JEQ | BPF_X:
923 case BPF_JMP32 | BPF_JEQ | BPF_X:
924 case BPF_JMP | BPF_JGT | BPF_X:
925 case BPF_JMP32 | BPF_JGT | BPF_X:
926 case BPF_JMP | BPF_JLT | BPF_X:
927 case BPF_JMP32 | BPF_JLT | BPF_X:
928 case BPF_JMP | BPF_JGE | BPF_X:
929 case BPF_JMP32 | BPF_JGE | BPF_X:
930 case BPF_JMP | BPF_JLE | BPF_X:
931 case BPF_JMP32 | BPF_JLE | BPF_X:
932 case BPF_JMP | BPF_JNE | BPF_X:
933 case BPF_JMP32 | BPF_JNE | BPF_X:
934 case BPF_JMP | BPF_JSGT | BPF_X:
935 case BPF_JMP32 | BPF_JSGT | BPF_X:
936 case BPF_JMP | BPF_JSLT | BPF_X:
937 case BPF_JMP32 | BPF_JSLT | BPF_X:
938 case BPF_JMP | BPF_JSGE | BPF_X:
939 case BPF_JMP32 | BPF_JSGE | BPF_X:
940 case BPF_JMP | BPF_JSLE | BPF_X:
941 case BPF_JMP32 | BPF_JSLE | BPF_X:
942 case BPF_JMP | BPF_JSET | BPF_X:
943 case BPF_JMP32 | BPF_JSET | BPF_X:
944 rvoff = rv_offset(i, off, ctx);
945 if (!is64) {
946 s = ctx->ninsns;
947 if (is_signed_bpf_cond(BPF_OP(code)))
948 emit_sext_32_rd_rs(&rd, &rs, ctx);
949 else
950 emit_zext_32_rd_rs(&rd, &rs, ctx);
951 e = ctx->ninsns;
952
953 /* Adjust for extra insns */
954 rvoff -= ninsns_rvoff(e - s);
955 }
956
957 if (BPF_OP(code) == BPF_JSET) {
958 /* Adjust for and */
959 rvoff -= 4;
960 emit_and(RV_REG_T1, rd, rs, ctx);
961 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
962 ctx);
963 } else {
964 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
965 }
966 break;
967
968 /* IF (dst COND imm) JUMP off */
969 case BPF_JMP | BPF_JEQ | BPF_K:
970 case BPF_JMP32 | BPF_JEQ | BPF_K:
971 case BPF_JMP | BPF_JGT | BPF_K:
972 case BPF_JMP32 | BPF_JGT | BPF_K:
973 case BPF_JMP | BPF_JLT | BPF_K:
974 case BPF_JMP32 | BPF_JLT | BPF_K:
975 case BPF_JMP | BPF_JGE | BPF_K:
976 case BPF_JMP32 | BPF_JGE | BPF_K:
977 case BPF_JMP | BPF_JLE | BPF_K:
978 case BPF_JMP32 | BPF_JLE | BPF_K:
979 case BPF_JMP | BPF_JNE | BPF_K:
980 case BPF_JMP32 | BPF_JNE | BPF_K:
981 case BPF_JMP | BPF_JSGT | BPF_K:
982 case BPF_JMP32 | BPF_JSGT | BPF_K:
983 case BPF_JMP | BPF_JSLT | BPF_K:
984 case BPF_JMP32 | BPF_JSLT | BPF_K:
985 case BPF_JMP | BPF_JSGE | BPF_K:
986 case BPF_JMP32 | BPF_JSGE | BPF_K:
987 case BPF_JMP | BPF_JSLE | BPF_K:
988 case BPF_JMP32 | BPF_JSLE | BPF_K:
989 rvoff = rv_offset(i, off, ctx);
990 s = ctx->ninsns;
991 if (imm) {
992 emit_imm(RV_REG_T1, imm, ctx);
993 rs = RV_REG_T1;
994 } else {
995 /* If imm is 0, simply use zero register. */
996 rs = RV_REG_ZERO;
997 }
998 if (!is64) {
999 if (is_signed_bpf_cond(BPF_OP(code)))
1000 emit_sext_32_rd(&rd, ctx);
1001 else
1002 emit_zext_32_rd_t1(&rd, ctx);
1003 }
1004 e = ctx->ninsns;
1005
1006 /* Adjust for extra insns */
1007 rvoff -= ninsns_rvoff(e - s);
1008 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
1009 break;
1010
1011 case BPF_JMP | BPF_JSET | BPF_K:
1012 case BPF_JMP32 | BPF_JSET | BPF_K:
1013 rvoff = rv_offset(i, off, ctx);
1014 s = ctx->ninsns;
1015 if (is_12b_int(imm)) {
1016 emit_andi(RV_REG_T1, rd, imm, ctx);
1017 } else {
1018 emit_imm(RV_REG_T1, imm, ctx);
1019 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx);
1020 }
1021 /* For jset32, we should clear the upper 32 bits of t1, but
1022 * sign-extension is sufficient here and saves one instruction,
1023 * as t1 is used only in comparison against zero.
1024 */
1025 if (!is64 && imm < 0)
1026 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx);
1027 e = ctx->ninsns;
1028 rvoff -= ninsns_rvoff(e - s);
1029 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
1030 break;
1031
1032 /* function call */
1033 case BPF_JMP | BPF_CALL:
1034 {
1035 bool fixed;
1036 u64 addr;
1037
1038 mark_call(ctx);
1039 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1040 &fixed);
1041 if (ret < 0)
1042 return ret;
1043 ret = emit_call(fixed, addr, ctx);
1044 if (ret)
1045 return ret;
1046 break;
1047 }
1048 /* tail call */
1049 case BPF_JMP | BPF_TAIL_CALL:
1050 if (emit_bpf_tail_call(i, ctx))
1051 return -1;
1052 break;
1053
1054 /* function return */
1055 case BPF_JMP | BPF_EXIT:
1056 if (i == ctx->prog->len - 1)
1057 break;
1058
1059 rvoff = epilogue_offset(ctx);
1060 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1061 if (ret)
1062 return ret;
1063 break;
1064
1065 /* dst = imm64 */
1066 case BPF_LD | BPF_IMM | BPF_DW:
1067 {
1068 struct bpf_insn insn1 = insn[1];
1069 u64 imm64;
1070
1071 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1072 if (bpf_pseudo_func(insn)) {
1073 /* fixed-length insns for extra jit pass */
1074 ret = emit_addr(rd, imm64, extra_pass, ctx);
1075 if (ret)
1076 return ret;
1077 } else {
1078 emit_imm(rd, imm64, ctx);
1079 }
1080
1081 return 1;
1082 }
1083
1084 /* LDX: dst = *(size *)(src + off) */
1085 case BPF_LDX | BPF_MEM | BPF_B:
1086 case BPF_LDX | BPF_MEM | BPF_H:
1087 case BPF_LDX | BPF_MEM | BPF_W:
1088 case BPF_LDX | BPF_MEM | BPF_DW:
1089 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1090 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1091 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1092 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1093 {
1094 int insn_len, insns_start;
1095
1096 switch (BPF_SIZE(code)) {
1097 case BPF_B:
1098 if (is_12b_int(off)) {
1099 insns_start = ctx->ninsns;
1100 emit(rv_lbu(rd, off, rs), ctx);
1101 insn_len = ctx->ninsns - insns_start;
1102 break;
1103 }
1104
1105 emit_imm(RV_REG_T1, off, ctx);
1106 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1107 insns_start = ctx->ninsns;
1108 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1109 insn_len = ctx->ninsns - insns_start;
1110 if (insn_is_zext(&insn[1]))
1111 return 1;
1112 break;
1113 case BPF_H:
1114 if (is_12b_int(off)) {
1115 insns_start = ctx->ninsns;
1116 emit(rv_lhu(rd, off, rs), ctx);
1117 insn_len = ctx->ninsns - insns_start;
1118 break;
1119 }
1120
1121 emit_imm(RV_REG_T1, off, ctx);
1122 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1123 insns_start = ctx->ninsns;
1124 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1125 insn_len = ctx->ninsns - insns_start;
1126 if (insn_is_zext(&insn[1]))
1127 return 1;
1128 break;
1129 case BPF_W:
1130 if (is_12b_int(off)) {
1131 insns_start = ctx->ninsns;
1132 emit(rv_lwu(rd, off, rs), ctx);
1133 insn_len = ctx->ninsns - insns_start;
1134 break;
1135 }
1136
1137 emit_imm(RV_REG_T1, off, ctx);
1138 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1139 insns_start = ctx->ninsns;
1140 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1141 insn_len = ctx->ninsns - insns_start;
1142 if (insn_is_zext(&insn[1]))
1143 return 1;
1144 break;
1145 case BPF_DW:
1146 if (is_12b_int(off)) {
1147 insns_start = ctx->ninsns;
1148 emit_ld(rd, off, rs, ctx);
1149 insn_len = ctx->ninsns - insns_start;
1150 break;
1151 }
1152
1153 emit_imm(RV_REG_T1, off, ctx);
1154 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
1155 insns_start = ctx->ninsns;
1156 emit_ld(rd, 0, RV_REG_T1, ctx);
1157 insn_len = ctx->ninsns - insns_start;
1158 break;
1159 }
1160
1161 ret = add_exception_handler(insn, ctx, rd, insn_len);
1162 if (ret)
1163 return ret;
1164 break;
1165 }
1166 /* speculation barrier */
1167 case BPF_ST | BPF_NOSPEC:
1168 break;
1169
1170 /* ST: *(size *)(dst + off) = imm */
1171 case BPF_ST | BPF_MEM | BPF_B:
1172 emit_imm(RV_REG_T1, imm, ctx);
1173 if (is_12b_int(off)) {
1174 emit(rv_sb(rd, off, RV_REG_T1), ctx);
1175 break;
1176 }
1177
1178 emit_imm(RV_REG_T2, off, ctx);
1179 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1180 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
1181 break;
1182
1183 case BPF_ST | BPF_MEM | BPF_H:
1184 emit_imm(RV_REG_T1, imm, ctx);
1185 if (is_12b_int(off)) {
1186 emit(rv_sh(rd, off, RV_REG_T1), ctx);
1187 break;
1188 }
1189
1190 emit_imm(RV_REG_T2, off, ctx);
1191 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1192 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
1193 break;
1194 case BPF_ST | BPF_MEM | BPF_W:
1195 emit_imm(RV_REG_T1, imm, ctx);
1196 if (is_12b_int(off)) {
1197 emit_sw(rd, off, RV_REG_T1, ctx);
1198 break;
1199 }
1200
1201 emit_imm(RV_REG_T2, off, ctx);
1202 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1203 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx);
1204 break;
1205 case BPF_ST | BPF_MEM | BPF_DW:
1206 emit_imm(RV_REG_T1, imm, ctx);
1207 if (is_12b_int(off)) {
1208 emit_sd(rd, off, RV_REG_T1, ctx);
1209 break;
1210 }
1211
1212 emit_imm(RV_REG_T2, off, ctx);
1213 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
1214 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx);
1215 break;
1216
1217 /* STX: *(size *)(dst + off) = src */
1218 case BPF_STX | BPF_MEM | BPF_B:
1219 if (is_12b_int(off)) {
1220 emit(rv_sb(rd, off, rs), ctx);
1221 break;
1222 }
1223
1224 emit_imm(RV_REG_T1, off, ctx);
1225 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1226 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
1227 break;
1228 case BPF_STX | BPF_MEM | BPF_H:
1229 if (is_12b_int(off)) {
1230 emit(rv_sh(rd, off, rs), ctx);
1231 break;
1232 }
1233
1234 emit_imm(RV_REG_T1, off, ctx);
1235 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1236 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
1237 break;
1238 case BPF_STX | BPF_MEM | BPF_W:
1239 if (is_12b_int(off)) {
1240 emit_sw(rd, off, rs, ctx);
1241 break;
1242 }
1243
1244 emit_imm(RV_REG_T1, off, ctx);
1245 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1246 emit_sw(RV_REG_T1, 0, rs, ctx);
1247 break;
1248 case BPF_STX | BPF_MEM | BPF_DW:
1249 if (is_12b_int(off)) {
1250 emit_sd(rd, off, rs, ctx);
1251 break;
1252 }
1253
1254 emit_imm(RV_REG_T1, off, ctx);
1255 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
1256 emit_sd(RV_REG_T1, 0, rs, ctx);
1257 break;
1258 case BPF_STX | BPF_ATOMIC | BPF_W:
1259 case BPF_STX | BPF_ATOMIC | BPF_DW:
1260 emit_atomic(rd, rs, off, imm,
1261 BPF_SIZE(code) == BPF_DW, ctx);
1262 break;
1263 default:
1264 pr_err("bpf-jit: unknown opcode %02x\n", code);
1265 return -EINVAL;
1266 }
1267
1268 return 0;
1269}
1270
1271void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1272{
1273 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1274
1275 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1276 if (bpf_stack_adjust)
1277 mark_fp(ctx);
1278
1279 if (seen_reg(RV_REG_RA, ctx))
1280 stack_adjust += 8;
1281 stack_adjust += 8; /* RV_REG_FP */
1282 if (seen_reg(RV_REG_S1, ctx))
1283 stack_adjust += 8;
1284 if (seen_reg(RV_REG_S2, ctx))
1285 stack_adjust += 8;
1286 if (seen_reg(RV_REG_S3, ctx))
1287 stack_adjust += 8;
1288 if (seen_reg(RV_REG_S4, ctx))
1289 stack_adjust += 8;
1290 if (seen_reg(RV_REG_S5, ctx))
1291 stack_adjust += 8;
1292 if (seen_reg(RV_REG_S6, ctx))
1293 stack_adjust += 8;
1294
1295 stack_adjust = round_up(stack_adjust, 16);
1296 stack_adjust += bpf_stack_adjust;
1297
1298 store_offset = stack_adjust - 8;
1299
1300 /* First instruction is always setting the tail-call-counter
1301 * (TCC) register. This instruction is skipped for tail calls.
1302 * Force using a 4-byte (non-compressed) instruction.
1303 */
1304 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1305
1306 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx);
1307
1308 if (seen_reg(RV_REG_RA, ctx)) {
1309 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx);
1310 store_offset -= 8;
1311 }
1312 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx);
1313 store_offset -= 8;
1314 if (seen_reg(RV_REG_S1, ctx)) {
1315 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx);
1316 store_offset -= 8;
1317 }
1318 if (seen_reg(RV_REG_S2, ctx)) {
1319 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx);
1320 store_offset -= 8;
1321 }
1322 if (seen_reg(RV_REG_S3, ctx)) {
1323 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx);
1324 store_offset -= 8;
1325 }
1326 if (seen_reg(RV_REG_S4, ctx)) {
1327 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx);
1328 store_offset -= 8;
1329 }
1330 if (seen_reg(RV_REG_S5, ctx)) {
1331 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx);
1332 store_offset -= 8;
1333 }
1334 if (seen_reg(RV_REG_S6, ctx)) {
1335 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx);
1336 store_offset -= 8;
1337 }
1338
1339 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx);
1340
1341 if (bpf_stack_adjust)
1342 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx);
1343
1344 /* Program contains calls and tail calls, so RV_REG_TCC need
1345 * to be saved across calls.
1346 */
1347 if (seen_tail_call(ctx) && seen_call(ctx))
1348 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx);
1349
1350 ctx->stack_size = stack_adjust;
1351}
1352
1353void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1354{
1355 __build_epilogue(false, ctx);
1356}