Loading...
Note: File does not exist in v3.5.6.
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9#ifndef _BPF_JIT_H
10#define _BPF_JIT_H
11
12#include <linux/bpf.h>
13#include <linux/filter.h>
14#include <asm/cacheflush.h>
15
16static inline bool rvc_enabled(void)
17{
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
19}
20
21enum {
22 RV_REG_ZERO = 0, /* The constant value 0 */
23 RV_REG_RA = 1, /* Return address */
24 RV_REG_SP = 2, /* Stack pointer */
25 RV_REG_GP = 3, /* Global pointer */
26 RV_REG_TP = 4, /* Thread pointer */
27 RV_REG_T0 = 5, /* Temporaries */
28 RV_REG_T1 = 6,
29 RV_REG_T2 = 7,
30 RV_REG_FP = 8, /* Saved register/frame pointer */
31 RV_REG_S1 = 9, /* Saved register */
32 RV_REG_A0 = 10, /* Function argument/return values */
33 RV_REG_A1 = 11, /* Function arguments */
34 RV_REG_A2 = 12,
35 RV_REG_A3 = 13,
36 RV_REG_A4 = 14,
37 RV_REG_A5 = 15,
38 RV_REG_A6 = 16,
39 RV_REG_A7 = 17,
40 RV_REG_S2 = 18, /* Saved registers */
41 RV_REG_S3 = 19,
42 RV_REG_S4 = 20,
43 RV_REG_S5 = 21,
44 RV_REG_S6 = 22,
45 RV_REG_S7 = 23,
46 RV_REG_S8 = 24,
47 RV_REG_S9 = 25,
48 RV_REG_S10 = 26,
49 RV_REG_S11 = 27,
50 RV_REG_T3 = 28, /* Temporaries */
51 RV_REG_T4 = 29,
52 RV_REG_T5 = 30,
53 RV_REG_T6 = 31,
54};
55
56static inline bool is_creg(u8 reg)
57{
58 return (1 << reg) & (BIT(RV_REG_FP) |
59 BIT(RV_REG_S1) |
60 BIT(RV_REG_A0) |
61 BIT(RV_REG_A1) |
62 BIT(RV_REG_A2) |
63 BIT(RV_REG_A3) |
64 BIT(RV_REG_A4) |
65 BIT(RV_REG_A5));
66}
67
68struct rv_jit_context {
69 struct bpf_prog *prog;
70 u16 *insns; /* RV insns */
71 int ninsns;
72 int epilogue_offset;
73 int *offset; /* BPF to RV */
74 unsigned long flags;
75 int stack_size;
76};
77
78/* Convert from ninsns to bytes. */
79static inline int ninsns_rvoff(int ninsns)
80{
81 return ninsns << 1;
82}
83
84struct rv_jit_data {
85 struct bpf_binary_header *header;
86 u8 *image;
87 struct rv_jit_context ctx;
88};
89
90static inline void bpf_fill_ill_insns(void *area, unsigned int size)
91{
92 memset(area, 0, size);
93}
94
95static inline void bpf_flush_icache(void *start, void *end)
96{
97 flush_icache_range((unsigned long)start, (unsigned long)end);
98}
99
100/* Emit a 4-byte riscv instruction. */
101static inline void emit(const u32 insn, struct rv_jit_context *ctx)
102{
103 if (ctx->insns) {
104 ctx->insns[ctx->ninsns] = insn;
105 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
106 }
107
108 ctx->ninsns += 2;
109}
110
111/* Emit a 2-byte riscv compressed instruction. */
112static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
113{
114 BUILD_BUG_ON(!rvc_enabled());
115
116 if (ctx->insns)
117 ctx->insns[ctx->ninsns] = insn;
118
119 ctx->ninsns++;
120}
121
122static inline int epilogue_offset(struct rv_jit_context *ctx)
123{
124 int to = ctx->epilogue_offset, from = ctx->ninsns;
125
126 return ninsns_rvoff(to - from);
127}
128
129/* Return -1 or inverted cond. */
130static inline int invert_bpf_cond(u8 cond)
131{
132 switch (cond) {
133 case BPF_JEQ:
134 return BPF_JNE;
135 case BPF_JGT:
136 return BPF_JLE;
137 case BPF_JLT:
138 return BPF_JGE;
139 case BPF_JGE:
140 return BPF_JLT;
141 case BPF_JLE:
142 return BPF_JGT;
143 case BPF_JNE:
144 return BPF_JEQ;
145 case BPF_JSGT:
146 return BPF_JSLE;
147 case BPF_JSLT:
148 return BPF_JSGE;
149 case BPF_JSGE:
150 return BPF_JSLT;
151 case BPF_JSLE:
152 return BPF_JSGT;
153 }
154 return -1;
155}
156
157static inline bool is_6b_int(long val)
158{
159 return -(1L << 5) <= val && val < (1L << 5);
160}
161
162static inline bool is_7b_uint(unsigned long val)
163{
164 return val < (1UL << 7);
165}
166
167static inline bool is_8b_uint(unsigned long val)
168{
169 return val < (1UL << 8);
170}
171
172static inline bool is_9b_uint(unsigned long val)
173{
174 return val < (1UL << 9);
175}
176
177static inline bool is_10b_int(long val)
178{
179 return -(1L << 9) <= val && val < (1L << 9);
180}
181
182static inline bool is_10b_uint(unsigned long val)
183{
184 return val < (1UL << 10);
185}
186
187static inline bool is_12b_int(long val)
188{
189 return -(1L << 11) <= val && val < (1L << 11);
190}
191
192static inline int is_12b_check(int off, int insn)
193{
194 if (!is_12b_int(off)) {
195 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
196 insn, (int)off);
197 return -1;
198 }
199 return 0;
200}
201
202static inline bool is_13b_int(long val)
203{
204 return -(1L << 12) <= val && val < (1L << 12);
205}
206
207static inline bool is_21b_int(long val)
208{
209 return -(1L << 20) <= val && val < (1L << 20);
210}
211
212static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
213{
214 int from, to;
215
216 off++; /* BPF branch is from PC+1, RV is from PC */
217 from = (insn > 0) ? ctx->offset[insn - 1] : 0;
218 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
219 return ninsns_rvoff(to - from);
220}
221
222/* Instruction formats. */
223
224static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
225 u8 opcode)
226{
227 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
228 (rd << 7) | opcode;
229}
230
231static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
232{
233 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
234 opcode;
235}
236
237static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
238{
239 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
240
241 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
242 (imm4_0 << 7) | opcode;
243}
244
245static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
246{
247 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
248 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
249
250 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
251 (imm4_1 << 7) | opcode;
252}
253
254static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
255{
256 return (imm31_12 << 12) | (rd << 7) | opcode;
257}
258
259static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
260{
261 u32 imm;
262
263 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
264 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
265
266 return (imm << 12) | (rd << 7) | opcode;
267}
268
269static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
270 u8 funct3, u8 rd, u8 opcode)
271{
272 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
273
274 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
275}
276
277/* RISC-V compressed instruction formats. */
278
279static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
280{
281 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
282}
283
284static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
285{
286 u32 imm;
287
288 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
289 return (funct3 << 13) | (rd << 7) | op | imm;
290}
291
292static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
293{
294 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
295}
296
297static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
298{
299 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
300}
301
302static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
303 u8 op)
304{
305 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
306 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
307}
308
309static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
310 u8 op)
311{
312 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
313 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
314}
315
316static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
317{
318 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
319 ((rs2 & 0x7) << 2) | op;
320}
321
322static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
323{
324 u32 imm;
325
326 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
327 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
328}
329
330/* Instructions shared by both RV32 and RV64. */
331
332static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
333{
334 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
335}
336
337static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
338{
339 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
340}
341
342static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
343{
344 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
345}
346
347static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
348{
349 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
350}
351
352static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
353{
354 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
355}
356
357static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
358{
359 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
360}
361
362static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
363{
364 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
365}
366
367static inline u32 rv_lui(u8 rd, u32 imm31_12)
368{
369 return rv_u_insn(imm31_12, rd, 0x37);
370}
371
372static inline u32 rv_auipc(u8 rd, u32 imm31_12)
373{
374 return rv_u_insn(imm31_12, rd, 0x17);
375}
376
377static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
378{
379 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
380}
381
382static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
383{
384 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
385}
386
387static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
388{
389 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
390}
391
392static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
393{
394 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
395}
396
397static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
398{
399 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
400}
401
402static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
403{
404 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
405}
406
407static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
408{
409 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
410}
411
412static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
413{
414 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
415}
416
417static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
418{
419 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
420}
421
422static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
423{
424 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
425}
426
427static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
428{
429 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
430}
431
432static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
433{
434 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
435}
436
437static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
438{
439 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
440}
441
442static inline u32 rv_jal(u8 rd, u32 imm20_1)
443{
444 return rv_j_insn(imm20_1, rd, 0x6f);
445}
446
447static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
448{
449 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
450}
451
452static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
453{
454 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
455}
456
457static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
458{
459 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
460}
461
462static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
463{
464 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
465}
466
467static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
468{
469 return rv_bltu(rs2, rs1, imm12_1);
470}
471
472static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
473{
474 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
475}
476
477static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
478{
479 return rv_bgeu(rs2, rs1, imm12_1);
480}
481
482static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
483{
484 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
485}
486
487static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
488{
489 return rv_blt(rs2, rs1, imm12_1);
490}
491
492static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
493{
494 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
495}
496
497static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
498{
499 return rv_bge(rs2, rs1, imm12_1);
500}
501
502static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
503{
504 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
505}
506
507static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
508{
509 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
510}
511
512static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
513{
514 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
515}
516
517static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
518{
519 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
520}
521
522static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
523{
524 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
525}
526
527static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
528{
529 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
530}
531
532static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
533{
534 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
535}
536
537/* RVC instrutions. */
538
539static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
540{
541 u32 imm;
542
543 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
544 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
545 return rv_ciw_insn(0x0, imm, rd, 0x0);
546}
547
548static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
549{
550 u32 imm_hi, imm_lo;
551
552 imm_hi = (imm7 & 0x38) >> 3;
553 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
554 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
555}
556
557static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
558{
559 u32 imm_hi, imm_lo;
560
561 imm_hi = (imm7 & 0x38) >> 3;
562 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
563 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
564}
565
566static inline u16 rvc_addi(u8 rd, u32 imm6)
567{
568 return rv_ci_insn(0, imm6, rd, 0x1);
569}
570
571static inline u16 rvc_li(u8 rd, u32 imm6)
572{
573 return rv_ci_insn(0x2, imm6, rd, 0x1);
574}
575
576static inline u16 rvc_addi16sp(u32 imm10)
577{
578 u32 imm;
579
580 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
581 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
582 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
583}
584
585static inline u16 rvc_lui(u8 rd, u32 imm6)
586{
587 return rv_ci_insn(0x3, imm6, rd, 0x1);
588}
589
590static inline u16 rvc_srli(u8 rd, u32 imm6)
591{
592 return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
593}
594
595static inline u16 rvc_srai(u8 rd, u32 imm6)
596{
597 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
598}
599
600static inline u16 rvc_andi(u8 rd, u32 imm6)
601{
602 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
603}
604
605static inline u16 rvc_sub(u8 rd, u8 rs)
606{
607 return rv_ca_insn(0x23, rd, 0, rs, 0x1);
608}
609
610static inline u16 rvc_xor(u8 rd, u8 rs)
611{
612 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
613}
614
615static inline u16 rvc_or(u8 rd, u8 rs)
616{
617 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
618}
619
620static inline u16 rvc_and(u8 rd, u8 rs)
621{
622 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
623}
624
625static inline u16 rvc_slli(u8 rd, u32 imm6)
626{
627 return rv_ci_insn(0, imm6, rd, 0x2);
628}
629
630static inline u16 rvc_lwsp(u8 rd, u32 imm8)
631{
632 u32 imm;
633
634 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
635 return rv_ci_insn(0x2, imm, rd, 0x2);
636}
637
638static inline u16 rvc_jr(u8 rs1)
639{
640 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
641}
642
643static inline u16 rvc_mv(u8 rd, u8 rs)
644{
645 return rv_cr_insn(0x8, rd, rs, 0x2);
646}
647
648static inline u16 rvc_jalr(u8 rs1)
649{
650 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
651}
652
653static inline u16 rvc_add(u8 rd, u8 rs)
654{
655 return rv_cr_insn(0x9, rd, rs, 0x2);
656}
657
658static inline u16 rvc_swsp(u32 imm8, u8 rs2)
659{
660 u32 imm;
661
662 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
663 return rv_css_insn(0x6, imm, rs2, 0x2);
664}
665
666/*
667 * RV64-only instructions.
668 *
669 * These instructions are not available on RV32. Wrap them below a #if to
670 * ensure that the RV32 JIT doesn't emit any of these instructions.
671 */
672
673#if __riscv_xlen == 64
674
675static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
676{
677 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
678}
679
680static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
681{
682 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
683}
684
685static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
686{
687 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
688}
689
690static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
691{
692 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
693}
694
695static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
696{
697 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
698}
699
700static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
701{
702 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
703}
704
705static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
706{
707 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
708}
709
710static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
711{
712 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
713}
714
715static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
716{
717 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
718}
719
720static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
721{
722 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
723}
724
725static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
726{
727 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
728}
729
730static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
731{
732 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
733}
734
735static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
736{
737 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
738}
739
740static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
741{
742 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
743}
744
745static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
746{
747 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
748}
749
750static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
751{
752 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
753}
754
755/* RV64-only RVC instructions. */
756
757static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
758{
759 u32 imm_hi, imm_lo;
760
761 imm_hi = (imm8 & 0x38) >> 3;
762 imm_lo = (imm8 & 0xc0) >> 6;
763 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
764}
765
766static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
767{
768 u32 imm_hi, imm_lo;
769
770 imm_hi = (imm8 & 0x38) >> 3;
771 imm_lo = (imm8 & 0xc0) >> 6;
772 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
773}
774
775static inline u16 rvc_subw(u8 rd, u8 rs)
776{
777 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
778}
779
780static inline u16 rvc_addiw(u8 rd, u32 imm6)
781{
782 return rv_ci_insn(0x1, imm6, rd, 0x1);
783}
784
785static inline u16 rvc_ldsp(u8 rd, u32 imm9)
786{
787 u32 imm;
788
789 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
790 return rv_ci_insn(0x3, imm, rd, 0x2);
791}
792
793static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
794{
795 u32 imm;
796
797 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
798 return rv_css_insn(0x7, imm, rs2, 0x2);
799}
800
801#endif /* __riscv_xlen == 64 */
802
803/* Helper functions that emit RVC instructions when possible. */
804
805static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
806{
807 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
808 emitc(rvc_jalr(rs), ctx);
809 else if (rvc_enabled() && !rd && rs && !imm)
810 emitc(rvc_jr(rs), ctx);
811 else
812 emit(rv_jalr(rd, rs, imm), ctx);
813}
814
815static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
816{
817 if (rvc_enabled() && rd && rs)
818 emitc(rvc_mv(rd, rs), ctx);
819 else
820 emit(rv_addi(rd, rs, 0), ctx);
821}
822
823static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
824{
825 if (rvc_enabled() && rd && rd == rs1 && rs2)
826 emitc(rvc_add(rd, rs2), ctx);
827 else
828 emit(rv_add(rd, rs1, rs2), ctx);
829}
830
831static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
832{
833 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
834 emitc(rvc_addi16sp(imm), ctx);
835 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
836 !(imm & 0x3) && imm)
837 emitc(rvc_addi4spn(rd, imm), ctx);
838 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
839 emitc(rvc_addi(rd, imm), ctx);
840 else
841 emit(rv_addi(rd, rs, imm), ctx);
842}
843
844static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
845{
846 if (rvc_enabled() && rd && is_6b_int(imm))
847 emitc(rvc_li(rd, imm), ctx);
848 else
849 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
850}
851
852static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
853{
854 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
855 emitc(rvc_lui(rd, imm), ctx);
856 else
857 emit(rv_lui(rd, imm), ctx);
858}
859
860static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
861{
862 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
863 emitc(rvc_slli(rd, imm), ctx);
864 else
865 emit(rv_slli(rd, rs, imm), ctx);
866}
867
868static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
869{
870 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
871 emitc(rvc_andi(rd, imm), ctx);
872 else
873 emit(rv_andi(rd, rs, imm), ctx);
874}
875
876static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
877{
878 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
879 emitc(rvc_srli(rd, imm), ctx);
880 else
881 emit(rv_srli(rd, rs, imm), ctx);
882}
883
884static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
885{
886 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
887 emitc(rvc_srai(rd, imm), ctx);
888 else
889 emit(rv_srai(rd, rs, imm), ctx);
890}
891
892static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
893{
894 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
895 emitc(rvc_sub(rd, rs2), ctx);
896 else
897 emit(rv_sub(rd, rs1, rs2), ctx);
898}
899
900static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
901{
902 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
903 emitc(rvc_or(rd, rs2), ctx);
904 else
905 emit(rv_or(rd, rs1, rs2), ctx);
906}
907
908static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
909{
910 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
911 emitc(rvc_and(rd, rs2), ctx);
912 else
913 emit(rv_and(rd, rs1, rs2), ctx);
914}
915
916static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
917{
918 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
919 emitc(rvc_xor(rd, rs2), ctx);
920 else
921 emit(rv_xor(rd, rs1, rs2), ctx);
922}
923
924static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
925{
926 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
927 emitc(rvc_lwsp(rd, off), ctx);
928 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
929 emitc(rvc_lw(rd, off, rs1), ctx);
930 else
931 emit(rv_lw(rd, off, rs1), ctx);
932}
933
934static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
935{
936 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
937 emitc(rvc_swsp(off, rs2), ctx);
938 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
939 emitc(rvc_sw(rs1, off, rs2), ctx);
940 else
941 emit(rv_sw(rs1, off, rs2), ctx);
942}
943
944/* RV64-only helper functions. */
945#if __riscv_xlen == 64
946
947static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
948{
949 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
950 emitc(rvc_addiw(rd, imm), ctx);
951 else
952 emit(rv_addiw(rd, rs, imm), ctx);
953}
954
955static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
956{
957 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
958 emitc(rvc_ldsp(rd, off), ctx);
959 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
960 emitc(rvc_ld(rd, off, rs1), ctx);
961 else
962 emit(rv_ld(rd, off, rs1), ctx);
963}
964
965static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
966{
967 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
968 emitc(rvc_sdsp(off, rs2), ctx);
969 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
970 emitc(rvc_sd(rs1, off, rs2), ctx);
971 else
972 emit(rv_sd(rs1, off, rs2), ctx);
973}
974
975static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
976{
977 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
978 emitc(rvc_subw(rd, rs2), ctx);
979 else
980 emit(rv_subw(rd, rs1, rs2), ctx);
981}
982
983#endif /* __riscv_xlen == 64 */
984
985void bpf_jit_build_prologue(struct rv_jit_context *ctx);
986void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
987
988int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
989 bool extra_pass);
990
991#endif /* _BPF_JIT_H */