Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9#ifndef _BPF_JIT_H
10#define _BPF_JIT_H
11
12#include <linux/bpf.h>
13#include <linux/filter.h>
14#include <asm/cacheflush.h>
15
16static inline bool rvc_enabled(void)
17{
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
19}
20
21enum {
22 RV_REG_ZERO = 0, /* The constant value 0 */
23 RV_REG_RA = 1, /* Return address */
24 RV_REG_SP = 2, /* Stack pointer */
25 RV_REG_GP = 3, /* Global pointer */
26 RV_REG_TP = 4, /* Thread pointer */
27 RV_REG_T0 = 5, /* Temporaries */
28 RV_REG_T1 = 6,
29 RV_REG_T2 = 7,
30 RV_REG_FP = 8, /* Saved register/frame pointer */
31 RV_REG_S1 = 9, /* Saved register */
32 RV_REG_A0 = 10, /* Function argument/return values */
33 RV_REG_A1 = 11, /* Function arguments */
34 RV_REG_A2 = 12,
35 RV_REG_A3 = 13,
36 RV_REG_A4 = 14,
37 RV_REG_A5 = 15,
38 RV_REG_A6 = 16,
39 RV_REG_A7 = 17,
40 RV_REG_S2 = 18, /* Saved registers */
41 RV_REG_S3 = 19,
42 RV_REG_S4 = 20,
43 RV_REG_S5 = 21,
44 RV_REG_S6 = 22,
45 RV_REG_S7 = 23,
46 RV_REG_S8 = 24,
47 RV_REG_S9 = 25,
48 RV_REG_S10 = 26,
49 RV_REG_S11 = 27,
50 RV_REG_T3 = 28, /* Temporaries */
51 RV_REG_T4 = 29,
52 RV_REG_T5 = 30,
53 RV_REG_T6 = 31,
54};
55
56static inline bool is_creg(u8 reg)
57{
58 return (1 << reg) & (BIT(RV_REG_FP) |
59 BIT(RV_REG_S1) |
60 BIT(RV_REG_A0) |
61 BIT(RV_REG_A1) |
62 BIT(RV_REG_A2) |
63 BIT(RV_REG_A3) |
64 BIT(RV_REG_A4) |
65 BIT(RV_REG_A5));
66}
67
68struct rv_jit_context {
69 struct bpf_prog *prog;
70 u16 *insns; /* RV insns */
71 int ninsns;
72 int body_len;
73 int epilogue_offset;
74 int *offset; /* BPF to RV */
75 int nexentries;
76 unsigned long flags;
77 int stack_size;
78};
79
80/* Convert from ninsns to bytes. */
81static inline int ninsns_rvoff(int ninsns)
82{
83 return ninsns << 1;
84}
85
86struct rv_jit_data {
87 struct bpf_binary_header *header;
88 u8 *image;
89 struct rv_jit_context ctx;
90};
91
92static inline void bpf_fill_ill_insns(void *area, unsigned int size)
93{
94 memset(area, 0, size);
95}
96
97static inline void bpf_flush_icache(void *start, void *end)
98{
99 flush_icache_range((unsigned long)start, (unsigned long)end);
100}
101
102/* Emit a 4-byte riscv instruction. */
103static inline void emit(const u32 insn, struct rv_jit_context *ctx)
104{
105 if (ctx->insns) {
106 ctx->insns[ctx->ninsns] = insn;
107 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
108 }
109
110 ctx->ninsns += 2;
111}
112
113/* Emit a 2-byte riscv compressed instruction. */
114static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
115{
116 BUILD_BUG_ON(!rvc_enabled());
117
118 if (ctx->insns)
119 ctx->insns[ctx->ninsns] = insn;
120
121 ctx->ninsns++;
122}
123
124static inline int epilogue_offset(struct rv_jit_context *ctx)
125{
126 int to = ctx->epilogue_offset, from = ctx->ninsns;
127
128 return ninsns_rvoff(to - from);
129}
130
131/* Return -1 or inverted cond. */
132static inline int invert_bpf_cond(u8 cond)
133{
134 switch (cond) {
135 case BPF_JEQ:
136 return BPF_JNE;
137 case BPF_JGT:
138 return BPF_JLE;
139 case BPF_JLT:
140 return BPF_JGE;
141 case BPF_JGE:
142 return BPF_JLT;
143 case BPF_JLE:
144 return BPF_JGT;
145 case BPF_JNE:
146 return BPF_JEQ;
147 case BPF_JSGT:
148 return BPF_JSLE;
149 case BPF_JSLT:
150 return BPF_JSGE;
151 case BPF_JSGE:
152 return BPF_JSLT;
153 case BPF_JSLE:
154 return BPF_JSGT;
155 }
156 return -1;
157}
158
159static inline bool is_6b_int(long val)
160{
161 return -(1L << 5) <= val && val < (1L << 5);
162}
163
164static inline bool is_7b_uint(unsigned long val)
165{
166 return val < (1UL << 7);
167}
168
169static inline bool is_8b_uint(unsigned long val)
170{
171 return val < (1UL << 8);
172}
173
174static inline bool is_9b_uint(unsigned long val)
175{
176 return val < (1UL << 9);
177}
178
179static inline bool is_10b_int(long val)
180{
181 return -(1L << 9) <= val && val < (1L << 9);
182}
183
184static inline bool is_10b_uint(unsigned long val)
185{
186 return val < (1UL << 10);
187}
188
189static inline bool is_12b_int(long val)
190{
191 return -(1L << 11) <= val && val < (1L << 11);
192}
193
194static inline int is_12b_check(int off, int insn)
195{
196 if (!is_12b_int(off)) {
197 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
198 insn, (int)off);
199 return -1;
200 }
201 return 0;
202}
203
204static inline bool is_13b_int(long val)
205{
206 return -(1L << 12) <= val && val < (1L << 12);
207}
208
209static inline bool is_21b_int(long val)
210{
211 return -(1L << 20) <= val && val < (1L << 20);
212}
213
214static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
215{
216 int from, to;
217
218 off++; /* BPF branch is from PC+1, RV is from PC */
219 from = (insn > 0) ? ctx->offset[insn - 1] : 0;
220 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
221 return ninsns_rvoff(to - from);
222}
223
224/* Instruction formats. */
225
226static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
227 u8 opcode)
228{
229 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
230 (rd << 7) | opcode;
231}
232
233static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
234{
235 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
236 opcode;
237}
238
239static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
240{
241 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
242
243 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
244 (imm4_0 << 7) | opcode;
245}
246
247static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
248{
249 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
250 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
251
252 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
253 (imm4_1 << 7) | opcode;
254}
255
256static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
257{
258 return (imm31_12 << 12) | (rd << 7) | opcode;
259}
260
261static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
262{
263 u32 imm;
264
265 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
266 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
267
268 return (imm << 12) | (rd << 7) | opcode;
269}
270
271static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
272 u8 funct3, u8 rd, u8 opcode)
273{
274 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
275
276 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
277}
278
279/* RISC-V compressed instruction formats. */
280
281static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
282{
283 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
284}
285
286static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
287{
288 u32 imm;
289
290 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
291 return (funct3 << 13) | (rd << 7) | op | imm;
292}
293
294static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
295{
296 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
297}
298
299static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
300{
301 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
302}
303
304static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
305 u8 op)
306{
307 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
308 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
309}
310
311static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
312 u8 op)
313{
314 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
315 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
316}
317
318static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
319{
320 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
321 ((rs2 & 0x7) << 2) | op;
322}
323
324static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
325{
326 u32 imm;
327
328 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
329 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
330}
331
332/* Instructions shared by both RV32 and RV64. */
333
334static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
335{
336 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
337}
338
339static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
340{
341 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
342}
343
344static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
345{
346 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
347}
348
349static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
350{
351 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
352}
353
354static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
355{
356 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
357}
358
359static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
360{
361 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
362}
363
364static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
365{
366 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
367}
368
369static inline u32 rv_lui(u8 rd, u32 imm31_12)
370{
371 return rv_u_insn(imm31_12, rd, 0x37);
372}
373
374static inline u32 rv_auipc(u8 rd, u32 imm31_12)
375{
376 return rv_u_insn(imm31_12, rd, 0x17);
377}
378
379static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
380{
381 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
382}
383
384static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
385{
386 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
387}
388
389static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
390{
391 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
392}
393
394static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
395{
396 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
397}
398
399static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
400{
401 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
402}
403
404static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
405{
406 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
407}
408
409static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
410{
411 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
412}
413
414static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
415{
416 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
417}
418
419static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
420{
421 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
422}
423
424static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
425{
426 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
427}
428
429static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
430{
431 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
432}
433
434static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
435{
436 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
437}
438
439static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
440{
441 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
442}
443
444static inline u32 rv_jal(u8 rd, u32 imm20_1)
445{
446 return rv_j_insn(imm20_1, rd, 0x6f);
447}
448
449static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
450{
451 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
452}
453
454static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
455{
456 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
457}
458
459static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
460{
461 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
462}
463
464static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
465{
466 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
467}
468
469static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
470{
471 return rv_bltu(rs2, rs1, imm12_1);
472}
473
474static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
475{
476 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
477}
478
479static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
480{
481 return rv_bgeu(rs2, rs1, imm12_1);
482}
483
484static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
485{
486 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
487}
488
489static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
490{
491 return rv_blt(rs2, rs1, imm12_1);
492}
493
494static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
495{
496 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
497}
498
499static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
500{
501 return rv_bge(rs2, rs1, imm12_1);
502}
503
504static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
505{
506 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
507}
508
509static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
510{
511 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
512}
513
514static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
515{
516 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
517}
518
519static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
520{
521 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
522}
523
524static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
525{
526 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
527}
528
529static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
530{
531 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
532}
533
534static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
535{
536 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
537}
538
539static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
540{
541 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
542}
543
544static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
545{
546 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
547}
548
549static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
550{
551 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
552}
553
554static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
555{
556 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
557}
558
559static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
560{
561 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
562}
563
564static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
565{
566 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
567}
568
569static inline u32 rv_fence(u8 pred, u8 succ)
570{
571 u16 imm11_0 = pred << 4 | succ;
572
573 return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
574}
575
576/* RVC instrutions. */
577
578static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
579{
580 u32 imm;
581
582 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
583 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
584 return rv_ciw_insn(0x0, imm, rd, 0x0);
585}
586
587static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
588{
589 u32 imm_hi, imm_lo;
590
591 imm_hi = (imm7 & 0x38) >> 3;
592 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
593 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
594}
595
596static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
597{
598 u32 imm_hi, imm_lo;
599
600 imm_hi = (imm7 & 0x38) >> 3;
601 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
602 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
603}
604
605static inline u16 rvc_addi(u8 rd, u32 imm6)
606{
607 return rv_ci_insn(0, imm6, rd, 0x1);
608}
609
610static inline u16 rvc_li(u8 rd, u32 imm6)
611{
612 return rv_ci_insn(0x2, imm6, rd, 0x1);
613}
614
615static inline u16 rvc_addi16sp(u32 imm10)
616{
617 u32 imm;
618
619 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
620 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
621 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
622}
623
624static inline u16 rvc_lui(u8 rd, u32 imm6)
625{
626 return rv_ci_insn(0x3, imm6, rd, 0x1);
627}
628
629static inline u16 rvc_srli(u8 rd, u32 imm6)
630{
631 return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
632}
633
634static inline u16 rvc_srai(u8 rd, u32 imm6)
635{
636 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
637}
638
639static inline u16 rvc_andi(u8 rd, u32 imm6)
640{
641 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
642}
643
644static inline u16 rvc_sub(u8 rd, u8 rs)
645{
646 return rv_ca_insn(0x23, rd, 0, rs, 0x1);
647}
648
649static inline u16 rvc_xor(u8 rd, u8 rs)
650{
651 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
652}
653
654static inline u16 rvc_or(u8 rd, u8 rs)
655{
656 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
657}
658
659static inline u16 rvc_and(u8 rd, u8 rs)
660{
661 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
662}
663
664static inline u16 rvc_slli(u8 rd, u32 imm6)
665{
666 return rv_ci_insn(0, imm6, rd, 0x2);
667}
668
669static inline u16 rvc_lwsp(u8 rd, u32 imm8)
670{
671 u32 imm;
672
673 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
674 return rv_ci_insn(0x2, imm, rd, 0x2);
675}
676
677static inline u16 rvc_jr(u8 rs1)
678{
679 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
680}
681
682static inline u16 rvc_mv(u8 rd, u8 rs)
683{
684 return rv_cr_insn(0x8, rd, rs, 0x2);
685}
686
687static inline u16 rvc_jalr(u8 rs1)
688{
689 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
690}
691
692static inline u16 rvc_add(u8 rd, u8 rs)
693{
694 return rv_cr_insn(0x9, rd, rs, 0x2);
695}
696
697static inline u16 rvc_swsp(u32 imm8, u8 rs2)
698{
699 u32 imm;
700
701 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
702 return rv_css_insn(0x6, imm, rs2, 0x2);
703}
704
705/*
706 * RV64-only instructions.
707 *
708 * These instructions are not available on RV32. Wrap them below a #if to
709 * ensure that the RV32 JIT doesn't emit any of these instructions.
710 */
711
712#if __riscv_xlen == 64
713
714static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
715{
716 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
717}
718
719static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
720{
721 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
722}
723
724static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
725{
726 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
727}
728
729static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
730{
731 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
732}
733
734static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
735{
736 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
737}
738
739static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
740{
741 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
742}
743
744static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
745{
746 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
747}
748
749static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
750{
751 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
752}
753
754static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
755{
756 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
757}
758
759static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
760{
761 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
762}
763
764static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
765{
766 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
767}
768
769static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
770{
771 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
772}
773
774static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
775{
776 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
777}
778
779static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
780{
781 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
782}
783
784static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
785{
786 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
787}
788
789static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
790{
791 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
792}
793
794static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
795{
796 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
797}
798
799static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
800{
801 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
802}
803
804static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
805{
806 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
807}
808
809static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
810{
811 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
812}
813
814static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
815{
816 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
817}
818
819static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
820{
821 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
822}
823
824/* RV64-only RVC instructions. */
825
826static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
827{
828 u32 imm_hi, imm_lo;
829
830 imm_hi = (imm8 & 0x38) >> 3;
831 imm_lo = (imm8 & 0xc0) >> 6;
832 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
833}
834
835static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
836{
837 u32 imm_hi, imm_lo;
838
839 imm_hi = (imm8 & 0x38) >> 3;
840 imm_lo = (imm8 & 0xc0) >> 6;
841 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
842}
843
844static inline u16 rvc_subw(u8 rd, u8 rs)
845{
846 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
847}
848
849static inline u16 rvc_addiw(u8 rd, u32 imm6)
850{
851 return rv_ci_insn(0x1, imm6, rd, 0x1);
852}
853
854static inline u16 rvc_ldsp(u8 rd, u32 imm9)
855{
856 u32 imm;
857
858 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
859 return rv_ci_insn(0x3, imm, rd, 0x2);
860}
861
862static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
863{
864 u32 imm;
865
866 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
867 return rv_css_insn(0x7, imm, rs2, 0x2);
868}
869
870#endif /* __riscv_xlen == 64 */
871
872/* Helper functions that emit RVC instructions when possible. */
873
874static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
875{
876 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
877 emitc(rvc_jalr(rs), ctx);
878 else if (rvc_enabled() && !rd && rs && !imm)
879 emitc(rvc_jr(rs), ctx);
880 else
881 emit(rv_jalr(rd, rs, imm), ctx);
882}
883
884static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
885{
886 if (rvc_enabled() && rd && rs)
887 emitc(rvc_mv(rd, rs), ctx);
888 else
889 emit(rv_addi(rd, rs, 0), ctx);
890}
891
892static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
893{
894 if (rvc_enabled() && rd && rd == rs1 && rs2)
895 emitc(rvc_add(rd, rs2), ctx);
896 else
897 emit(rv_add(rd, rs1, rs2), ctx);
898}
899
900static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
901{
902 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
903 emitc(rvc_addi16sp(imm), ctx);
904 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
905 !(imm & 0x3) && imm)
906 emitc(rvc_addi4spn(rd, imm), ctx);
907 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
908 emitc(rvc_addi(rd, imm), ctx);
909 else
910 emit(rv_addi(rd, rs, imm), ctx);
911}
912
913static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
914{
915 if (rvc_enabled() && rd && is_6b_int(imm))
916 emitc(rvc_li(rd, imm), ctx);
917 else
918 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
919}
920
921static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
922{
923 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
924 emitc(rvc_lui(rd, imm), ctx);
925 else
926 emit(rv_lui(rd, imm), ctx);
927}
928
929static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
930{
931 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
932 emitc(rvc_slli(rd, imm), ctx);
933 else
934 emit(rv_slli(rd, rs, imm), ctx);
935}
936
937static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
938{
939 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
940 emitc(rvc_andi(rd, imm), ctx);
941 else
942 emit(rv_andi(rd, rs, imm), ctx);
943}
944
945static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
946{
947 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
948 emitc(rvc_srli(rd, imm), ctx);
949 else
950 emit(rv_srli(rd, rs, imm), ctx);
951}
952
953static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
954{
955 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
956 emitc(rvc_srai(rd, imm), ctx);
957 else
958 emit(rv_srai(rd, rs, imm), ctx);
959}
960
961static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
962{
963 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
964 emitc(rvc_sub(rd, rs2), ctx);
965 else
966 emit(rv_sub(rd, rs1, rs2), ctx);
967}
968
969static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
970{
971 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
972 emitc(rvc_or(rd, rs2), ctx);
973 else
974 emit(rv_or(rd, rs1, rs2), ctx);
975}
976
977static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
978{
979 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
980 emitc(rvc_and(rd, rs2), ctx);
981 else
982 emit(rv_and(rd, rs1, rs2), ctx);
983}
984
985static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
986{
987 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
988 emitc(rvc_xor(rd, rs2), ctx);
989 else
990 emit(rv_xor(rd, rs1, rs2), ctx);
991}
992
993static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
994{
995 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
996 emitc(rvc_lwsp(rd, off), ctx);
997 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
998 emitc(rvc_lw(rd, off, rs1), ctx);
999 else
1000 emit(rv_lw(rd, off, rs1), ctx);
1001}
1002
1003static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1004{
1005 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1006 emitc(rvc_swsp(off, rs2), ctx);
1007 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1008 emitc(rvc_sw(rs1, off, rs2), ctx);
1009 else
1010 emit(rv_sw(rs1, off, rs2), ctx);
1011}
1012
1013/* RV64-only helper functions. */
1014#if __riscv_xlen == 64
1015
1016static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1017{
1018 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1019 emitc(rvc_addiw(rd, imm), ctx);
1020 else
1021 emit(rv_addiw(rd, rs, imm), ctx);
1022}
1023
1024static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1025{
1026 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1027 emitc(rvc_ldsp(rd, off), ctx);
1028 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1029 emitc(rvc_ld(rd, off, rs1), ctx);
1030 else
1031 emit(rv_ld(rd, off, rs1), ctx);
1032}
1033
1034static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1035{
1036 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1037 emitc(rvc_sdsp(off, rs2), ctx);
1038 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1039 emitc(rvc_sd(rs1, off, rs2), ctx);
1040 else
1041 emit(rv_sd(rs1, off, rs2), ctx);
1042}
1043
1044static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1045{
1046 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1047 emitc(rvc_subw(rd, rs2), ctx);
1048 else
1049 emit(rv_subw(rd, rs1, rs2), ctx);
1050}
1051
1052#endif /* __riscv_xlen == 64 */
1053
1054void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1055void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1056
1057int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1058 bool extra_pass);
1059
1060#endif /* _BPF_JIT_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9#ifndef _BPF_JIT_H
10#define _BPF_JIT_H
11
12#include <linux/bpf.h>
13#include <linux/filter.h>
14#include <asm/cacheflush.h>
15
16static inline bool rvc_enabled(void)
17{
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
19}
20
21enum {
22 RV_REG_ZERO = 0, /* The constant value 0 */
23 RV_REG_RA = 1, /* Return address */
24 RV_REG_SP = 2, /* Stack pointer */
25 RV_REG_GP = 3, /* Global pointer */
26 RV_REG_TP = 4, /* Thread pointer */
27 RV_REG_T0 = 5, /* Temporaries */
28 RV_REG_T1 = 6,
29 RV_REG_T2 = 7,
30 RV_REG_FP = 8, /* Saved register/frame pointer */
31 RV_REG_S1 = 9, /* Saved register */
32 RV_REG_A0 = 10, /* Function argument/return values */
33 RV_REG_A1 = 11, /* Function arguments */
34 RV_REG_A2 = 12,
35 RV_REG_A3 = 13,
36 RV_REG_A4 = 14,
37 RV_REG_A5 = 15,
38 RV_REG_A6 = 16,
39 RV_REG_A7 = 17,
40 RV_REG_S2 = 18, /* Saved registers */
41 RV_REG_S3 = 19,
42 RV_REG_S4 = 20,
43 RV_REG_S5 = 21,
44 RV_REG_S6 = 22,
45 RV_REG_S7 = 23,
46 RV_REG_S8 = 24,
47 RV_REG_S9 = 25,
48 RV_REG_S10 = 26,
49 RV_REG_S11 = 27,
50 RV_REG_T3 = 28, /* Temporaries */
51 RV_REG_T4 = 29,
52 RV_REG_T5 = 30,
53 RV_REG_T6 = 31,
54};
55
56static inline bool is_creg(u8 reg)
57{
58 return (1 << reg) & (BIT(RV_REG_FP) |
59 BIT(RV_REG_S1) |
60 BIT(RV_REG_A0) |
61 BIT(RV_REG_A1) |
62 BIT(RV_REG_A2) |
63 BIT(RV_REG_A3) |
64 BIT(RV_REG_A4) |
65 BIT(RV_REG_A5));
66}
67
68struct rv_jit_context {
69 struct bpf_prog *prog;
70 u16 *insns; /* RV insns */
71 u16 *ro_insns;
72 int ninsns;
73 int prologue_len;
74 int epilogue_offset;
75 int *offset; /* BPF to RV */
76 int nexentries;
77 unsigned long flags;
78 int stack_size;
79};
80
81/* Convert from ninsns to bytes. */
82static inline int ninsns_rvoff(int ninsns)
83{
84 return ninsns << 1;
85}
86
87struct rv_jit_data {
88 struct bpf_binary_header *header;
89 struct bpf_binary_header *ro_header;
90 u8 *image;
91 u8 *ro_image;
92 struct rv_jit_context ctx;
93};
94
95static inline void bpf_fill_ill_insns(void *area, unsigned int size)
96{
97 memset(area, 0, size);
98}
99
100static inline void bpf_flush_icache(void *start, void *end)
101{
102 flush_icache_range((unsigned long)start, (unsigned long)end);
103}
104
105/* Emit a 4-byte riscv instruction. */
106static inline void emit(const u32 insn, struct rv_jit_context *ctx)
107{
108 if (ctx->insns) {
109 ctx->insns[ctx->ninsns] = insn;
110 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
111 }
112
113 ctx->ninsns += 2;
114}
115
116/* Emit a 2-byte riscv compressed instruction. */
117static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
118{
119 BUILD_BUG_ON(!rvc_enabled());
120
121 if (ctx->insns)
122 ctx->insns[ctx->ninsns] = insn;
123
124 ctx->ninsns++;
125}
126
127static inline int epilogue_offset(struct rv_jit_context *ctx)
128{
129 int to = ctx->epilogue_offset, from = ctx->ninsns;
130
131 return ninsns_rvoff(to - from);
132}
133
134/* Return -1 or inverted cond. */
135static inline int invert_bpf_cond(u8 cond)
136{
137 switch (cond) {
138 case BPF_JEQ:
139 return BPF_JNE;
140 case BPF_JGT:
141 return BPF_JLE;
142 case BPF_JLT:
143 return BPF_JGE;
144 case BPF_JGE:
145 return BPF_JLT;
146 case BPF_JLE:
147 return BPF_JGT;
148 case BPF_JNE:
149 return BPF_JEQ;
150 case BPF_JSGT:
151 return BPF_JSLE;
152 case BPF_JSLT:
153 return BPF_JSGE;
154 case BPF_JSGE:
155 return BPF_JSLT;
156 case BPF_JSLE:
157 return BPF_JSGT;
158 }
159 return -1;
160}
161
162static inline bool is_6b_int(long val)
163{
164 return -(1L << 5) <= val && val < (1L << 5);
165}
166
167static inline bool is_7b_uint(unsigned long val)
168{
169 return val < (1UL << 7);
170}
171
172static inline bool is_8b_uint(unsigned long val)
173{
174 return val < (1UL << 8);
175}
176
177static inline bool is_9b_uint(unsigned long val)
178{
179 return val < (1UL << 9);
180}
181
182static inline bool is_10b_int(long val)
183{
184 return -(1L << 9) <= val && val < (1L << 9);
185}
186
187static inline bool is_10b_uint(unsigned long val)
188{
189 return val < (1UL << 10);
190}
191
192static inline bool is_12b_int(long val)
193{
194 return -(1L << 11) <= val && val < (1L << 11);
195}
196
197static inline int is_12b_check(int off, int insn)
198{
199 if (!is_12b_int(off)) {
200 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
201 insn, (int)off);
202 return -1;
203 }
204 return 0;
205}
206
207static inline bool is_13b_int(long val)
208{
209 return -(1L << 12) <= val && val < (1L << 12);
210}
211
212static inline bool is_21b_int(long val)
213{
214 return -(1L << 20) <= val && val < (1L << 20);
215}
216
217static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
218{
219 int from, to;
220
221 off++; /* BPF branch is from PC+1, RV is from PC */
222 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len;
223 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len;
224 return ninsns_rvoff(to - from);
225}
226
227/* Instruction formats. */
228
229static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
230 u8 opcode)
231{
232 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
233 (rd << 7) | opcode;
234}
235
236static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
237{
238 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
239 opcode;
240}
241
242static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
243{
244 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
245
246 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
247 (imm4_0 << 7) | opcode;
248}
249
250static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
251{
252 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
253 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
254
255 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
256 (imm4_1 << 7) | opcode;
257}
258
259static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
260{
261 return (imm31_12 << 12) | (rd << 7) | opcode;
262}
263
264static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
265{
266 u32 imm;
267
268 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
269 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
270
271 return (imm << 12) | (rd << 7) | opcode;
272}
273
274static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
275 u8 funct3, u8 rd, u8 opcode)
276{
277 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
278
279 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
280}
281
282/* RISC-V compressed instruction formats. */
283
284static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
285{
286 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
287}
288
289static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
290{
291 u32 imm;
292
293 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
294 return (funct3 << 13) | (rd << 7) | op | imm;
295}
296
297static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
298{
299 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
300}
301
302static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
303{
304 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
305}
306
307static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
308 u8 op)
309{
310 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
311 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
312}
313
314static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
315 u8 op)
316{
317 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
318 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
319}
320
321static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
322{
323 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
324 ((rs2 & 0x7) << 2) | op;
325}
326
327static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
328{
329 u32 imm;
330
331 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
332 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
333}
334
335/* Instructions shared by both RV32 and RV64. */
336
337static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
338{
339 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
340}
341
342static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
343{
344 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
345}
346
347static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
348{
349 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
350}
351
352static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
353{
354 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
355}
356
357static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
358{
359 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
360}
361
362static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
363{
364 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
365}
366
367static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
368{
369 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
370}
371
372static inline u32 rv_lui(u8 rd, u32 imm31_12)
373{
374 return rv_u_insn(imm31_12, rd, 0x37);
375}
376
377static inline u32 rv_auipc(u8 rd, u32 imm31_12)
378{
379 return rv_u_insn(imm31_12, rd, 0x17);
380}
381
382static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
383{
384 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
385}
386
387static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
388{
389 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
390}
391
392static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
393{
394 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
395}
396
397static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
398{
399 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
400}
401
402static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
403{
404 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
405}
406
407static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
408{
409 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
410}
411
412static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
413{
414 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
415}
416
417static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
418{
419 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
420}
421
422static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
423{
424 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
425}
426
427static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
428{
429 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
430}
431
432static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
433{
434 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
435}
436
437static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
438{
439 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
440}
441
442static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
443{
444 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
445}
446
447static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
448{
449 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
450}
451
452static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
453{
454 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
455}
456
457static inline u32 rv_jal(u8 rd, u32 imm20_1)
458{
459 return rv_j_insn(imm20_1, rd, 0x6f);
460}
461
462static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
463{
464 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
465}
466
467static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
468{
469 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
470}
471
472static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
473{
474 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
475}
476
477static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
478{
479 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
480}
481
482static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
483{
484 return rv_bltu(rs2, rs1, imm12_1);
485}
486
487static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
488{
489 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
490}
491
492static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
493{
494 return rv_bgeu(rs2, rs1, imm12_1);
495}
496
497static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
498{
499 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
500}
501
502static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
503{
504 return rv_blt(rs2, rs1, imm12_1);
505}
506
507static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
508{
509 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
510}
511
512static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
513{
514 return rv_bge(rs2, rs1, imm12_1);
515}
516
517static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
518{
519 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
520}
521
522static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
523{
524 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
525}
526
527static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
528{
529 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
530}
531
532static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
533{
534 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
535}
536
537static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
538{
539 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
540}
541
542static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
543{
544 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
545}
546
547static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
548{
549 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
550}
551
552static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
553{
554 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
555}
556
557static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
558{
559 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
560}
561
562static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
563{
564 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
565}
566
567static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
568{
569 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
570}
571
572static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
573{
574 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
575}
576
577static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
578{
579 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
580}
581
582static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
583{
584 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
585}
586
587static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
588{
589 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
590}
591
592static inline u32 rv_fence(u8 pred, u8 succ)
593{
594 u16 imm11_0 = pred << 4 | succ;
595
596 return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
597}
598
599static inline u32 rv_nop(void)
600{
601 return rv_i_insn(0, 0, 0, 0, 0x13);
602}
603
604/* RVC instrutions. */
605
606static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
607{
608 u32 imm;
609
610 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
611 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
612 return rv_ciw_insn(0x0, imm, rd, 0x0);
613}
614
615static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
616{
617 u32 imm_hi, imm_lo;
618
619 imm_hi = (imm7 & 0x38) >> 3;
620 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
621 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
622}
623
624static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
625{
626 u32 imm_hi, imm_lo;
627
628 imm_hi = (imm7 & 0x38) >> 3;
629 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
630 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
631}
632
633static inline u16 rvc_addi(u8 rd, u32 imm6)
634{
635 return rv_ci_insn(0, imm6, rd, 0x1);
636}
637
638static inline u16 rvc_li(u8 rd, u32 imm6)
639{
640 return rv_ci_insn(0x2, imm6, rd, 0x1);
641}
642
643static inline u16 rvc_addi16sp(u32 imm10)
644{
645 u32 imm;
646
647 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
648 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
649 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
650}
651
652static inline u16 rvc_lui(u8 rd, u32 imm6)
653{
654 return rv_ci_insn(0x3, imm6, rd, 0x1);
655}
656
657static inline u16 rvc_srli(u8 rd, u32 imm6)
658{
659 return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
660}
661
662static inline u16 rvc_srai(u8 rd, u32 imm6)
663{
664 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
665}
666
667static inline u16 rvc_andi(u8 rd, u32 imm6)
668{
669 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
670}
671
672static inline u16 rvc_sub(u8 rd, u8 rs)
673{
674 return rv_ca_insn(0x23, rd, 0, rs, 0x1);
675}
676
677static inline u16 rvc_xor(u8 rd, u8 rs)
678{
679 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
680}
681
682static inline u16 rvc_or(u8 rd, u8 rs)
683{
684 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
685}
686
687static inline u16 rvc_and(u8 rd, u8 rs)
688{
689 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
690}
691
692static inline u16 rvc_slli(u8 rd, u32 imm6)
693{
694 return rv_ci_insn(0, imm6, rd, 0x2);
695}
696
697static inline u16 rvc_lwsp(u8 rd, u32 imm8)
698{
699 u32 imm;
700
701 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
702 return rv_ci_insn(0x2, imm, rd, 0x2);
703}
704
705static inline u16 rvc_jr(u8 rs1)
706{
707 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
708}
709
710static inline u16 rvc_mv(u8 rd, u8 rs)
711{
712 return rv_cr_insn(0x8, rd, rs, 0x2);
713}
714
715static inline u16 rvc_jalr(u8 rs1)
716{
717 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
718}
719
720static inline u16 rvc_add(u8 rd, u8 rs)
721{
722 return rv_cr_insn(0x9, rd, rs, 0x2);
723}
724
725static inline u16 rvc_swsp(u32 imm8, u8 rs2)
726{
727 u32 imm;
728
729 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
730 return rv_css_insn(0x6, imm, rs2, 0x2);
731}
732
733/*
734 * RV64-only instructions.
735 *
736 * These instructions are not available on RV32. Wrap them below a #if to
737 * ensure that the RV32 JIT doesn't emit any of these instructions.
738 */
739
740#if __riscv_xlen == 64
741
742static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
743{
744 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
745}
746
747static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
748{
749 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
750}
751
752static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
753{
754 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
755}
756
757static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
758{
759 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
760}
761
762static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
763{
764 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
765}
766
767static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
768{
769 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
770}
771
772static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
773{
774 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
775}
776
777static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
778{
779 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
780}
781
782static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
783{
784 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
785}
786
787static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
788{
789 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
790}
791
792static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
793{
794 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
795}
796
797static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
798{
799 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
800}
801
802static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
803{
804 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
805}
806
807static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
808{
809 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
810}
811
812static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
813{
814 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
815}
816
817static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
818{
819 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
820}
821
822static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
823{
824 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
825}
826
827static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
828{
829 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
830}
831
832static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
833{
834 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
835}
836
837static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
838{
839 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
840}
841
842static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
843{
844 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
845}
846
847static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
848{
849 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
850}
851
852static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
853{
854 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
855}
856
857static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
858{
859 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
860}
861
862/* RV64-only RVC instructions. */
863
864static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
865{
866 u32 imm_hi, imm_lo;
867
868 imm_hi = (imm8 & 0x38) >> 3;
869 imm_lo = (imm8 & 0xc0) >> 6;
870 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
871}
872
873static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
874{
875 u32 imm_hi, imm_lo;
876
877 imm_hi = (imm8 & 0x38) >> 3;
878 imm_lo = (imm8 & 0xc0) >> 6;
879 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
880}
881
882static inline u16 rvc_subw(u8 rd, u8 rs)
883{
884 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
885}
886
887static inline u16 rvc_addiw(u8 rd, u32 imm6)
888{
889 return rv_ci_insn(0x1, imm6, rd, 0x1);
890}
891
892static inline u16 rvc_ldsp(u8 rd, u32 imm9)
893{
894 u32 imm;
895
896 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
897 return rv_ci_insn(0x3, imm, rd, 0x2);
898}
899
900static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
901{
902 u32 imm;
903
904 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
905 return rv_css_insn(0x7, imm, rs2, 0x2);
906}
907
908#endif /* __riscv_xlen == 64 */
909
910/* Helper functions that emit RVC instructions when possible. */
911
912static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
913{
914 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
915 emitc(rvc_jalr(rs), ctx);
916 else if (rvc_enabled() && !rd && rs && !imm)
917 emitc(rvc_jr(rs), ctx);
918 else
919 emit(rv_jalr(rd, rs, imm), ctx);
920}
921
922static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
923{
924 if (rvc_enabled() && rd && rs)
925 emitc(rvc_mv(rd, rs), ctx);
926 else
927 emit(rv_addi(rd, rs, 0), ctx);
928}
929
930static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
931{
932 if (rvc_enabled() && rd && rd == rs1 && rs2)
933 emitc(rvc_add(rd, rs2), ctx);
934 else
935 emit(rv_add(rd, rs1, rs2), ctx);
936}
937
938static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
939{
940 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
941 emitc(rvc_addi16sp(imm), ctx);
942 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
943 !(imm & 0x3) && imm)
944 emitc(rvc_addi4spn(rd, imm), ctx);
945 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
946 emitc(rvc_addi(rd, imm), ctx);
947 else
948 emit(rv_addi(rd, rs, imm), ctx);
949}
950
951static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
952{
953 if (rvc_enabled() && rd && is_6b_int(imm))
954 emitc(rvc_li(rd, imm), ctx);
955 else
956 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
957}
958
959static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
960{
961 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
962 emitc(rvc_lui(rd, imm), ctx);
963 else
964 emit(rv_lui(rd, imm), ctx);
965}
966
967static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
968{
969 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
970 emitc(rvc_slli(rd, imm), ctx);
971 else
972 emit(rv_slli(rd, rs, imm), ctx);
973}
974
975static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
976{
977 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
978 emitc(rvc_andi(rd, imm), ctx);
979 else
980 emit(rv_andi(rd, rs, imm), ctx);
981}
982
983static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
984{
985 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
986 emitc(rvc_srli(rd, imm), ctx);
987 else
988 emit(rv_srli(rd, rs, imm), ctx);
989}
990
991static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
992{
993 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
994 emitc(rvc_srai(rd, imm), ctx);
995 else
996 emit(rv_srai(rd, rs, imm), ctx);
997}
998
999static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1000{
1001 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1002 emitc(rvc_sub(rd, rs2), ctx);
1003 else
1004 emit(rv_sub(rd, rs1, rs2), ctx);
1005}
1006
1007static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1008{
1009 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1010 emitc(rvc_or(rd, rs2), ctx);
1011 else
1012 emit(rv_or(rd, rs1, rs2), ctx);
1013}
1014
1015static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1016{
1017 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1018 emitc(rvc_and(rd, rs2), ctx);
1019 else
1020 emit(rv_and(rd, rs1, rs2), ctx);
1021}
1022
1023static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1024{
1025 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1026 emitc(rvc_xor(rd, rs2), ctx);
1027 else
1028 emit(rv_xor(rd, rs1, rs2), ctx);
1029}
1030
1031static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1032{
1033 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
1034 emitc(rvc_lwsp(rd, off), ctx);
1035 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
1036 emitc(rvc_lw(rd, off, rs1), ctx);
1037 else
1038 emit(rv_lw(rd, off, rs1), ctx);
1039}
1040
1041static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1042{
1043 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1044 emitc(rvc_swsp(off, rs2), ctx);
1045 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1046 emitc(rvc_sw(rs1, off, rs2), ctx);
1047 else
1048 emit(rv_sw(rs1, off, rs2), ctx);
1049}
1050
1051/* RV64-only helper functions. */
1052#if __riscv_xlen == 64
1053
1054static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1055{
1056 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1057 emitc(rvc_addiw(rd, imm), ctx);
1058 else
1059 emit(rv_addiw(rd, rs, imm), ctx);
1060}
1061
1062static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1063{
1064 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1065 emitc(rvc_ldsp(rd, off), ctx);
1066 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1067 emitc(rvc_ld(rd, off, rs1), ctx);
1068 else
1069 emit(rv_ld(rd, off, rs1), ctx);
1070}
1071
1072static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1073{
1074 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1075 emitc(rvc_sdsp(off, rs2), ctx);
1076 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1077 emitc(rvc_sd(rs1, off, rs2), ctx);
1078 else
1079 emit(rv_sd(rs1, off, rs2), ctx);
1080}
1081
1082static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1083{
1084 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1085 emitc(rvc_subw(rd, rs2), ctx);
1086 else
1087 emit(rv_subw(rd, rs1, rs2), ctx);
1088}
1089
1090#endif /* __riscv_xlen == 64 */
1091
1092void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1093void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1094
1095int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1096 bool extra_pass);
1097
1098#endif /* _BPF_JIT_H */