Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * eBPF JIT compiler for PPC32
4 *
5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
6 * CS GROUP France
7 *
8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
9 */
10#include <linux/moduleloader.h>
11#include <asm/cacheflush.h>
12#include <asm/asm-compat.h>
13#include <linux/netdevice.h>
14#include <linux/filter.h>
15#include <linux/if_vlan.h>
16#include <asm/kprobes.h>
17#include <linux/bpf.h>
18
19#include "bpf_jit.h"
20
21/*
22 * Stack layout:
23 *
24 * [ prev sp ] <-------------
25 * [ nv gpr save area ] 16 * 4 |
26 * fp (r31) --> [ ebpf stack space ] upto 512 |
27 * [ frame header ] 16 |
28 * sp (r1) ---> [ stack pointer ] --------------
29 */
30
31/* for gpr non volatile registers r17 to r31 (14) + tail call */
32#define BPF_PPC_STACK_SAVE (15 * 4 + 4)
33/* stack frame, ensure this is quadword aligned */
34#define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
35
36#define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
37
38/* PPC NVR range -- update this if we ever use NVRs below r17 */
39#define BPF_PPC_NVR_MIN _R17
40#define BPF_PPC_TC _R16
41
42/* BPF register usage */
43#define TMP_REG (MAX_BPF_JIT_REG + 0)
44
45/* BPF to ppc register mappings */
46void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
47{
48 /* function return value */
49 ctx->b2p[BPF_REG_0] = _R12;
50 /* function arguments */
51 ctx->b2p[BPF_REG_1] = _R4;
52 ctx->b2p[BPF_REG_2] = _R6;
53 ctx->b2p[BPF_REG_3] = _R8;
54 ctx->b2p[BPF_REG_4] = _R10;
55 ctx->b2p[BPF_REG_5] = _R22;
56 /* non volatile registers */
57 ctx->b2p[BPF_REG_6] = _R24;
58 ctx->b2p[BPF_REG_7] = _R26;
59 ctx->b2p[BPF_REG_8] = _R28;
60 ctx->b2p[BPF_REG_9] = _R30;
61 /* frame pointer aka BPF_REG_10 */
62 ctx->b2p[BPF_REG_FP] = _R18;
63 /* eBPF jit internal registers */
64 ctx->b2p[BPF_REG_AX] = _R20;
65 ctx->b2p[TMP_REG] = _R31; /* 32 bits */
66}
67
68static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
69{
70 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
71 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
72
73 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
74 /* Use the hole we have left for alignment */
75 return BPF_PPC_STACKFRAME(ctx) - 4;
76}
77
78#define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */
79#define SEEN_NVREG_FULL_MASK 0x0003ffff /* Non volatile registers r14-r31 */
80#define SEEN_NVREG_TEMP_MASK 0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */
81
82void bpf_jit_realloc_regs(struct codegen_context *ctx)
83{
84 unsigned int nvreg_mask;
85
86 if (ctx->seen & SEEN_FUNC)
87 nvreg_mask = SEEN_NVREG_TEMP_MASK;
88 else
89 nvreg_mask = SEEN_NVREG_FULL_MASK;
90
91 while (ctx->seen & nvreg_mask &&
92 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
93 int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab));
94 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
95 int i;
96
97 for (i = BPF_REG_0; i <= TMP_REG; i++) {
98 if (ctx->b2p[i] != old)
99 continue;
100 ctx->b2p[i] = new;
101 bpf_set_seen_register(ctx, new);
102 bpf_clear_seen_register(ctx, old);
103 if (i != TMP_REG) {
104 bpf_set_seen_register(ctx, new - 1);
105 bpf_clear_seen_register(ctx, old - 1);
106 }
107 break;
108 }
109 }
110}
111
112void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
113{
114 int i;
115
116 /* Initialize tail_call_cnt, to be skipped if we do tail calls. */
117 EMIT(PPC_RAW_LI(_R4, 0));
118
119#define BPF_TAILCALL_PROLOGUE_SIZE 4
120
121 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
122
123 if (ctx->seen & SEEN_TAILCALL)
124 EMIT(PPC_RAW_STW(_R4, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
125
126 /* First arg comes in as a 32 bits pointer. */
127 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3));
128 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0));
129
130 /*
131 * We need a stack frame, but we don't necessarily need to
132 * save/restore LR unless we call other functions
133 */
134 if (ctx->seen & SEEN_FUNC)
135 EMIT(PPC_RAW_MFLR(_R0));
136
137 /*
138 * Back up non-volatile regs -- registers r18-r31
139 */
140 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
141 if (bpf_is_seen_register(ctx, i))
142 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
143
144 /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
145 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
146 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8);
147 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12);
148 }
149
150 /* Setup frame pointer to point to the bpf stack area */
151 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) {
152 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0));
153 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
154 STACK_FRAME_MIN_SIZE + ctx->stack_size));
155 }
156
157 if (ctx->seen & SEEN_FUNC)
158 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
159}
160
161static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
162{
163 int i;
164
165 /* Restore NVRs */
166 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
167 if (bpf_is_seen_register(ctx, i))
168 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
169
170 if (ctx->seen & SEEN_FUNC)
171 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
172
173 /* Tear down our stack frame */
174 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
175
176 if (ctx->seen & SEEN_FUNC)
177 EMIT(PPC_RAW_MTLR(_R0));
178
179}
180
181void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
182{
183 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
184
185 bpf_jit_emit_common_epilogue(image, ctx);
186
187 EMIT(PPC_RAW_BLR());
188}
189
190int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
191{
192 s32 rel = (s32)func - (s32)(image + ctx->idx);
193
194 if (image && rel < 0x2000000 && rel >= -0x2000000) {
195 PPC_BL(func);
196 EMIT(PPC_RAW_NOP());
197 EMIT(PPC_RAW_NOP());
198 EMIT(PPC_RAW_NOP());
199 } else {
200 /* Load function address into r0 */
201 EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
202 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
203 EMIT(PPC_RAW_MTCTR(_R0));
204 EMIT(PPC_RAW_BCTRL());
205 }
206
207 return 0;
208}
209
210static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
211{
212 /*
213 * By now, the eBPF program has already setup parameters in r3-r6
214 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
215 * r5-r6/BPF_REG_2 - pointer to bpf_array
216 * r7-r8/BPF_REG_3 - index in bpf_array
217 */
218 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
219 int b2p_index = bpf_to_ppc(BPF_REG_3);
220
221 /*
222 * if (index >= array->map.max_entries)
223 * goto out;
224 */
225 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
226 EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
227 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
228 PPC_BCC_SHORT(COND_GE, out);
229
230 /*
231 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
232 * goto out;
233 */
234 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
235 /* tail_call_cnt++; */
236 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
237 PPC_BCC_SHORT(COND_GE, out);
238
239 /* prog = array->ptrs[index]; */
240 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
241 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
242 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
243
244 /*
245 * if (prog == NULL)
246 * goto out;
247 */
248 EMIT(PPC_RAW_CMPLWI(_R3, 0));
249 PPC_BCC_SHORT(COND_EQ, out);
250
251 /* goto *(prog->bpf_func + prologue_size); */
252 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
253 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
254 EMIT(PPC_RAW_MTCTR(_R3));
255
256 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1)));
257
258 /* Put tail_call_cnt in r4 */
259 EMIT(PPC_RAW_MR(_R4, _R0));
260
261 /* tear restore NVRs, ... */
262 bpf_jit_emit_common_epilogue(image, ctx);
263
264 EMIT(PPC_RAW_BCTR());
265
266 /* out: */
267 return 0;
268}
269
270/* Assemble the body code between the prologue & epilogue */
271int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
272 u32 *addrs, int pass)
273{
274 const struct bpf_insn *insn = fp->insnsi;
275 int flen = fp->len;
276 int i, ret;
277
278 /* Start of epilogue code - will only be valid 2nd pass onwards */
279 u32 exit_addr = addrs[flen];
280
281 for (i = 0; i < flen; i++) {
282 u32 code = insn[i].code;
283 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
284 u32 dst_reg_h = dst_reg - 1;
285 u32 src_reg = bpf_to_ppc(insn[i].src_reg);
286 u32 src_reg_h = src_reg - 1;
287 u32 ax_reg = bpf_to_ppc(BPF_REG_AX);
288 u32 tmp_reg = bpf_to_ppc(TMP_REG);
289 u32 size = BPF_SIZE(code);
290 u32 save_reg, ret_reg;
291 s16 off = insn[i].off;
292 s32 imm = insn[i].imm;
293 bool func_addr_fixed;
294 u64 func_addr;
295 u32 true_cond;
296 u32 tmp_idx;
297 int j;
298
299 /*
300 * addrs[] maps a BPF bytecode address into a real offset from
301 * the start of the body code.
302 */
303 addrs[i] = ctx->idx * 4;
304
305 /*
306 * As an optimization, we note down which registers
307 * are used so that we can only save/restore those in our
308 * prologue and epilogue. We do this here regardless of whether
309 * the actual BPF instruction uses src/dst registers or not
310 * (for instance, BPF_CALL does not use them). The expectation
311 * is that those instructions will have src_reg/dst_reg set to
312 * 0. Even otherwise, we just lose some prologue/epilogue
313 * optimization but everything else should work without
314 * any issues.
315 */
316 if (dst_reg >= 3 && dst_reg < 32) {
317 bpf_set_seen_register(ctx, dst_reg);
318 bpf_set_seen_register(ctx, dst_reg_h);
319 }
320
321 if (src_reg >= 3 && src_reg < 32) {
322 bpf_set_seen_register(ctx, src_reg);
323 bpf_set_seen_register(ctx, src_reg_h);
324 }
325
326 switch (code) {
327 /*
328 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
329 */
330 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
331 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
332 break;
333 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
334 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
335 EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
336 break;
337 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
338 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
339 break;
340 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
341 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
342 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
343 break;
344 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
345 imm = -imm;
346 fallthrough;
347 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
348 if (IMM_HA(imm) & 0xffff)
349 EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
350 if (IMM_L(imm))
351 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
352 break;
353 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
354 imm = -imm;
355 fallthrough;
356 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
357 if (!imm)
358 break;
359
360 if (imm >= -32768 && imm < 32768) {
361 EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
362 } else {
363 PPC_LI32(_R0, imm);
364 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0));
365 }
366 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
367 EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
368 else
369 EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
370 break;
371 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
372 bpf_set_seen_register(ctx, tmp_reg);
373 EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h));
374 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
375 EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
376 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
377 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
378 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
379 break;
380 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
381 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
382 break;
383 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
384 if (imm >= -32768 && imm < 32768) {
385 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
386 } else {
387 PPC_LI32(_R0, imm);
388 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0));
389 }
390 break;
391 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
392 if (!imm) {
393 PPC_LI32(dst_reg, 0);
394 PPC_LI32(dst_reg_h, 0);
395 break;
396 }
397 if (imm == 1)
398 break;
399 if (imm == -1) {
400 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
401 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
402 break;
403 }
404 bpf_set_seen_register(ctx, tmp_reg);
405 PPC_LI32(tmp_reg, imm);
406 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
407 if (imm < 0)
408 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
409 EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg));
410 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
411 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
412 break;
413 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
414 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
415 break;
416 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
417 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg));
418 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
419 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
420 break;
421 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
422 return -EOPNOTSUPP;
423 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
424 return -EOPNOTSUPP;
425 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
426 if (!imm)
427 return -EINVAL;
428 if (imm == 1)
429 break;
430
431 PPC_LI32(_R0, imm);
432 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0));
433 break;
434 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
435 if (!imm)
436 return -EINVAL;
437
438 if (!is_power_of_2((u32)imm)) {
439 bpf_set_seen_register(ctx, tmp_reg);
440 PPC_LI32(tmp_reg, imm);
441 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg));
442 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
443 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
444 break;
445 }
446 if (imm == 1)
447 EMIT(PPC_RAW_LI(dst_reg, 0));
448 else
449 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
450
451 break;
452 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
453 if (!imm)
454 return -EINVAL;
455 if (imm < 0)
456 imm = -imm;
457 if (!is_power_of_2(imm))
458 return -EOPNOTSUPP;
459 if (imm == 1)
460 EMIT(PPC_RAW_LI(dst_reg, 0));
461 else
462 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
463 EMIT(PPC_RAW_LI(dst_reg_h, 0));
464 break;
465 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
466 if (!imm)
467 return -EINVAL;
468 if (!is_power_of_2(abs(imm)))
469 return -EOPNOTSUPP;
470
471 if (imm < 0) {
472 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
473 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
474 imm = -imm;
475 }
476 if (imm == 1)
477 break;
478 imm = ilog2(imm);
479 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
480 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
481 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
482 break;
483 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
484 EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
485 break;
486 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
487 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
488 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
489 break;
490
491 /*
492 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
493 */
494 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
495 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
496 EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
497 break;
498 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
499 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
500 break;
501 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
502 if (imm >= 0)
503 EMIT(PPC_RAW_LI(dst_reg_h, 0));
504 fallthrough;
505 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
506 if (!IMM_H(imm)) {
507 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
508 } else if (!IMM_L(imm)) {
509 EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
510 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
511 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
512 32 - fls(imm), 32 - ffs(imm)));
513 } else {
514 PPC_LI32(_R0, imm);
515 EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0));
516 }
517 break;
518 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
519 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
520 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
521 break;
522 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
523 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
524 break;
525 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
526 /* Sign-extended */
527 if (imm < 0)
528 EMIT(PPC_RAW_LI(dst_reg_h, -1));
529 fallthrough;
530 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
531 if (IMM_L(imm))
532 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
533 if (IMM_H(imm))
534 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
535 break;
536 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
537 if (dst_reg == src_reg) {
538 EMIT(PPC_RAW_LI(dst_reg, 0));
539 EMIT(PPC_RAW_LI(dst_reg_h, 0));
540 } else {
541 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
542 EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
543 }
544 break;
545 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
546 if (dst_reg == src_reg)
547 EMIT(PPC_RAW_LI(dst_reg, 0));
548 else
549 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
550 break;
551 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
552 if (imm < 0)
553 EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
554 fallthrough;
555 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
556 if (IMM_L(imm))
557 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
558 if (IMM_H(imm))
559 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
560 break;
561 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
562 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
563 break;
564 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
565 bpf_set_seen_register(ctx, tmp_reg);
566 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
567 EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
568 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
569 EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0));
570 EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
571 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
572 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
573 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
574 break;
575 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */
576 if (!imm)
577 break;
578 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
579 break;
580 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */
581 if (imm < 0)
582 return -EINVAL;
583 if (!imm)
584 break;
585 if (imm < 32) {
586 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
587 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
588 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
589 break;
590 }
591 if (imm < 64)
592 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
593 else
594 EMIT(PPC_RAW_LI(dst_reg_h, 0));
595 EMIT(PPC_RAW_LI(dst_reg, 0));
596 break;
597 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
598 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
599 break;
600 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
601 bpf_set_seen_register(ctx, tmp_reg);
602 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
603 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
604 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
605 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
606 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
607 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
608 EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
609 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
610 break;
611 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
612 if (!imm)
613 break;
614 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
615 break;
616 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
617 if (imm < 0)
618 return -EINVAL;
619 if (!imm)
620 break;
621 if (imm < 32) {
622 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
623 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
624 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
625 break;
626 }
627 if (imm < 64)
628 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
629 else
630 EMIT(PPC_RAW_LI(dst_reg, 0));
631 EMIT(PPC_RAW_LI(dst_reg_h, 0));
632 break;
633 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
634 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
635 break;
636 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
637 bpf_set_seen_register(ctx, tmp_reg);
638 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
639 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
640 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
641 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
642 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
643 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
644 EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
645 EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
646 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
647 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
648 break;
649 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
650 if (!imm)
651 break;
652 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
653 break;
654 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
655 if (imm < 0)
656 return -EINVAL;
657 if (!imm)
658 break;
659 if (imm < 32) {
660 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
661 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
662 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
663 break;
664 }
665 if (imm < 64)
666 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
667 else
668 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
669 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
670 break;
671
672 /*
673 * MOV
674 */
675 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
676 if (dst_reg == src_reg)
677 break;
678 EMIT(PPC_RAW_MR(dst_reg, src_reg));
679 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
680 break;
681 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
682 /* special mov32 for zext */
683 if (imm == 1)
684 EMIT(PPC_RAW_LI(dst_reg_h, 0));
685 else if (dst_reg != src_reg)
686 EMIT(PPC_RAW_MR(dst_reg, src_reg));
687 break;
688 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
689 PPC_LI32(dst_reg, imm);
690 PPC_EX32(dst_reg_h, imm);
691 break;
692 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
693 PPC_LI32(dst_reg, imm);
694 break;
695
696 /*
697 * BPF_FROM_BE/LE
698 */
699 case BPF_ALU | BPF_END | BPF_FROM_LE:
700 switch (imm) {
701 case 16:
702 /* Copy 16 bits to upper part */
703 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
704 /* Rotate 8 bits right & mask */
705 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
706 break;
707 case 32:
708 /*
709 * Rotate word left by 8 bits:
710 * 2 bytes are already in their final position
711 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
712 */
713 EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31));
714 /* Rotate 24 bits and insert byte 1 */
715 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7));
716 /* Rotate 24 bits and insert byte 3 */
717 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23));
718 EMIT(PPC_RAW_MR(dst_reg, _R0));
719 break;
720 case 64:
721 bpf_set_seen_register(ctx, tmp_reg);
722 EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
723 EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31));
724 /* Rotate 24 bits and insert byte 1 */
725 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
726 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7));
727 /* Rotate 24 bits and insert byte 3 */
728 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
729 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23));
730 EMIT(PPC_RAW_MR(dst_reg, _R0));
731 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
732 break;
733 }
734 break;
735 case BPF_ALU | BPF_END | BPF_FROM_BE:
736 switch (imm) {
737 case 16:
738 /* zero-extend 16 bits into 32 bits */
739 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
740 break;
741 case 32:
742 case 64:
743 /* nop */
744 break;
745 }
746 break;
747
748 /*
749 * BPF_ST NOSPEC (speculation barrier)
750 */
751 case BPF_ST | BPF_NOSPEC:
752 break;
753
754 /*
755 * BPF_ST(X)
756 */
757 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
758 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
759 break;
760 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
761 PPC_LI32(_R0, imm);
762 EMIT(PPC_RAW_STB(_R0, dst_reg, off));
763 break;
764 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
765 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
766 break;
767 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
768 PPC_LI32(_R0, imm);
769 EMIT(PPC_RAW_STH(_R0, dst_reg, off));
770 break;
771 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
772 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
773 break;
774 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
775 PPC_LI32(_R0, imm);
776 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
777 break;
778 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
779 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
780 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
781 break;
782 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
783 PPC_LI32(_R0, imm);
784 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
785 PPC_EX32(_R0, imm);
786 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
787 break;
788
789 /*
790 * BPF_STX ATOMIC (atomic ops)
791 */
792 case BPF_STX | BPF_ATOMIC | BPF_W:
793 save_reg = _R0;
794 ret_reg = src_reg;
795
796 bpf_set_seen_register(ctx, tmp_reg);
797 bpf_set_seen_register(ctx, ax_reg);
798
799 /* Get offset into TMP_REG */
800 EMIT(PPC_RAW_LI(tmp_reg, off));
801 tmp_idx = ctx->idx * 4;
802 /* load value from memory into r0 */
803 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
804
805 /* Save old value in BPF_REG_AX */
806 if (imm & BPF_FETCH)
807 EMIT(PPC_RAW_MR(ax_reg, _R0));
808
809 switch (imm) {
810 case BPF_ADD:
811 case BPF_ADD | BPF_FETCH:
812 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
813 break;
814 case BPF_AND:
815 case BPF_AND | BPF_FETCH:
816 EMIT(PPC_RAW_AND(_R0, _R0, src_reg));
817 break;
818 case BPF_OR:
819 case BPF_OR | BPF_FETCH:
820 EMIT(PPC_RAW_OR(_R0, _R0, src_reg));
821 break;
822 case BPF_XOR:
823 case BPF_XOR | BPF_FETCH:
824 EMIT(PPC_RAW_XOR(_R0, _R0, src_reg));
825 break;
826 case BPF_CMPXCHG:
827 /*
828 * Return old value in BPF_REG_0 for BPF_CMPXCHG &
829 * in src_reg for other cases.
830 */
831 ret_reg = bpf_to_ppc(BPF_REG_0);
832
833 /* Compare with old value in BPF_REG_0 */
834 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0));
835 /* Don't set if different from old value */
836 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
837 fallthrough;
838 case BPF_XCHG:
839 save_reg = src_reg;
840 break;
841 default:
842 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
843 code, i);
844 return -EOPNOTSUPP;
845 }
846
847 /* store new value */
848 EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg));
849 /* we're done if this succeeded */
850 PPC_BCC_SHORT(COND_NE, tmp_idx);
851
852 /* For the BPF_FETCH variant, get old data into src_reg */
853 if (imm & BPF_FETCH) {
854 EMIT(PPC_RAW_MR(ret_reg, ax_reg));
855 if (!fp->aux->verifier_zext)
856 EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */
857 }
858 break;
859
860 case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */
861 return -EOPNOTSUPP;
862
863 /*
864 * BPF_LDX
865 */
866 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
867 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
868 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
869 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
870 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
871 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
872 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
873 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
874 /*
875 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
876 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
877 * load only if addr is kernel address (see is_kernel_addr()), otherwise
878 * set dst_reg=0 and move on.
879 */
880 if (BPF_MODE(code) == BPF_PROBE_MEM) {
881 PPC_LI32(_R0, TASK_SIZE - off);
882 EMIT(PPC_RAW_CMPLW(src_reg, _R0));
883 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4);
884 EMIT(PPC_RAW_LI(dst_reg, 0));
885 /*
886 * For BPF_DW case, "li reg_h,0" would be needed when
887 * !fp->aux->verifier_zext. Emit NOP otherwise.
888 *
889 * Note that "li reg_h,0" is emitted for BPF_B/H/W case,
890 * if necessary. So, jump there insted of emitting an
891 * additional "li reg_h,0" instruction.
892 */
893 if (size == BPF_DW && !fp->aux->verifier_zext)
894 EMIT(PPC_RAW_LI(dst_reg_h, 0));
895 else
896 EMIT(PPC_RAW_NOP());
897 /*
898 * Need to jump two instructions instead of one for BPF_DW case
899 * as there are two load instructions for dst_reg_h & dst_reg
900 * respectively.
901 */
902 if (size == BPF_DW)
903 PPC_JMP((ctx->idx + 3) * 4);
904 else
905 PPC_JMP((ctx->idx + 2) * 4);
906 }
907
908 switch (size) {
909 case BPF_B:
910 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
911 break;
912 case BPF_H:
913 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
914 break;
915 case BPF_W:
916 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
917 break;
918 case BPF_DW:
919 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
920 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
921 break;
922 }
923
924 if (size != BPF_DW && !fp->aux->verifier_zext)
925 EMIT(PPC_RAW_LI(dst_reg_h, 0));
926
927 if (BPF_MODE(code) == BPF_PROBE_MEM) {
928 int insn_idx = ctx->idx - 1;
929 int jmp_off = 4;
930
931 /*
932 * In case of BPF_DW, two lwz instructions are emitted, one
933 * for higher 32-bit and another for lower 32-bit. So, set
934 * ex->insn to the first of the two and jump over both
935 * instructions in fixup.
936 *
937 * Similarly, with !verifier_zext, two instructions are
938 * emitted for BPF_B/H/W case. So, set ex->insn to the
939 * instruction that could fault and skip over both
940 * instructions.
941 */
942 if (size == BPF_DW || !fp->aux->verifier_zext) {
943 insn_idx -= 1;
944 jmp_off += 4;
945 }
946
947 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
948 jmp_off, dst_reg);
949 if (ret)
950 return ret;
951 }
952 break;
953
954 /*
955 * Doubleword load
956 * 16 byte instruction that uses two 'struct bpf_insn'
957 */
958 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
959 tmp_idx = ctx->idx;
960 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
961 PPC_LI32(dst_reg, (u32)insn[i].imm);
962 /* padding to allow full 4 instructions for later patching */
963 for (j = ctx->idx - tmp_idx; j < 4; j++)
964 EMIT(PPC_RAW_NOP());
965 /* Adjust for two bpf instructions */
966 addrs[++i] = ctx->idx * 4;
967 break;
968
969 /*
970 * Return/Exit
971 */
972 case BPF_JMP | BPF_EXIT:
973 /*
974 * If this isn't the very last instruction, branch to
975 * the epilogue. If we _are_ the last instruction,
976 * we'll just fall through to the epilogue.
977 */
978 if (i != flen - 1) {
979 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
980 if (ret)
981 return ret;
982 }
983 /* else fall through to the epilogue */
984 break;
985
986 /*
987 * Call kernel helper or bpf function
988 */
989 case BPF_JMP | BPF_CALL:
990 ctx->seen |= SEEN_FUNC;
991
992 ret = bpf_jit_get_func_addr(fp, &insn[i], false,
993 &func_addr, &func_addr_fixed);
994 if (ret < 0)
995 return ret;
996
997 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
998 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8));
999 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
1000 }
1001
1002 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1003 if (ret)
1004 return ret;
1005
1006 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3));
1007 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4));
1008 break;
1009
1010 /*
1011 * Jumps and branches
1012 */
1013 case BPF_JMP | BPF_JA:
1014 PPC_JMP(addrs[i + 1 + off]);
1015 break;
1016
1017 case BPF_JMP | BPF_JGT | BPF_K:
1018 case BPF_JMP | BPF_JGT | BPF_X:
1019 case BPF_JMP | BPF_JSGT | BPF_K:
1020 case BPF_JMP | BPF_JSGT | BPF_X:
1021 case BPF_JMP32 | BPF_JGT | BPF_K:
1022 case BPF_JMP32 | BPF_JGT | BPF_X:
1023 case BPF_JMP32 | BPF_JSGT | BPF_K:
1024 case BPF_JMP32 | BPF_JSGT | BPF_X:
1025 true_cond = COND_GT;
1026 goto cond_branch;
1027 case BPF_JMP | BPF_JLT | BPF_K:
1028 case BPF_JMP | BPF_JLT | BPF_X:
1029 case BPF_JMP | BPF_JSLT | BPF_K:
1030 case BPF_JMP | BPF_JSLT | BPF_X:
1031 case BPF_JMP32 | BPF_JLT | BPF_K:
1032 case BPF_JMP32 | BPF_JLT | BPF_X:
1033 case BPF_JMP32 | BPF_JSLT | BPF_K:
1034 case BPF_JMP32 | BPF_JSLT | BPF_X:
1035 true_cond = COND_LT;
1036 goto cond_branch;
1037 case BPF_JMP | BPF_JGE | BPF_K:
1038 case BPF_JMP | BPF_JGE | BPF_X:
1039 case BPF_JMP | BPF_JSGE | BPF_K:
1040 case BPF_JMP | BPF_JSGE | BPF_X:
1041 case BPF_JMP32 | BPF_JGE | BPF_K:
1042 case BPF_JMP32 | BPF_JGE | BPF_X:
1043 case BPF_JMP32 | BPF_JSGE | BPF_K:
1044 case BPF_JMP32 | BPF_JSGE | BPF_X:
1045 true_cond = COND_GE;
1046 goto cond_branch;
1047 case BPF_JMP | BPF_JLE | BPF_K:
1048 case BPF_JMP | BPF_JLE | BPF_X:
1049 case BPF_JMP | BPF_JSLE | BPF_K:
1050 case BPF_JMP | BPF_JSLE | BPF_X:
1051 case BPF_JMP32 | BPF_JLE | BPF_K:
1052 case BPF_JMP32 | BPF_JLE | BPF_X:
1053 case BPF_JMP32 | BPF_JSLE | BPF_K:
1054 case BPF_JMP32 | BPF_JSLE | BPF_X:
1055 true_cond = COND_LE;
1056 goto cond_branch;
1057 case BPF_JMP | BPF_JEQ | BPF_K:
1058 case BPF_JMP | BPF_JEQ | BPF_X:
1059 case BPF_JMP32 | BPF_JEQ | BPF_K:
1060 case BPF_JMP32 | BPF_JEQ | BPF_X:
1061 true_cond = COND_EQ;
1062 goto cond_branch;
1063 case BPF_JMP | BPF_JNE | BPF_K:
1064 case BPF_JMP | BPF_JNE | BPF_X:
1065 case BPF_JMP32 | BPF_JNE | BPF_K:
1066 case BPF_JMP32 | BPF_JNE | BPF_X:
1067 true_cond = COND_NE;
1068 goto cond_branch;
1069 case BPF_JMP | BPF_JSET | BPF_K:
1070 case BPF_JMP | BPF_JSET | BPF_X:
1071 case BPF_JMP32 | BPF_JSET | BPF_K:
1072 case BPF_JMP32 | BPF_JSET | BPF_X:
1073 true_cond = COND_NE;
1074 /* fallthrough; */
1075
1076cond_branch:
1077 switch (code) {
1078 case BPF_JMP | BPF_JGT | BPF_X:
1079 case BPF_JMP | BPF_JLT | BPF_X:
1080 case BPF_JMP | BPF_JGE | BPF_X:
1081 case BPF_JMP | BPF_JLE | BPF_X:
1082 case BPF_JMP | BPF_JEQ | BPF_X:
1083 case BPF_JMP | BPF_JNE | BPF_X:
1084 /* unsigned comparison */
1085 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1086 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1087 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1088 break;
1089 case BPF_JMP32 | BPF_JGT | BPF_X:
1090 case BPF_JMP32 | BPF_JLT | BPF_X:
1091 case BPF_JMP32 | BPF_JGE | BPF_X:
1092 case BPF_JMP32 | BPF_JLE | BPF_X:
1093 case BPF_JMP32 | BPF_JEQ | BPF_X:
1094 case BPF_JMP32 | BPF_JNE | BPF_X:
1095 /* unsigned comparison */
1096 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1097 break;
1098 case BPF_JMP | BPF_JSGT | BPF_X:
1099 case BPF_JMP | BPF_JSLT | BPF_X:
1100 case BPF_JMP | BPF_JSGE | BPF_X:
1101 case BPF_JMP | BPF_JSLE | BPF_X:
1102 /* signed comparison */
1103 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1104 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1105 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1106 break;
1107 case BPF_JMP32 | BPF_JSGT | BPF_X:
1108 case BPF_JMP32 | BPF_JSLT | BPF_X:
1109 case BPF_JMP32 | BPF_JSGE | BPF_X:
1110 case BPF_JMP32 | BPF_JSLE | BPF_X:
1111 /* signed comparison */
1112 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1113 break;
1114 case BPF_JMP | BPF_JSET | BPF_X:
1115 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1116 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1117 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1118 break;
1119 case BPF_JMP32 | BPF_JSET | BPF_X: {
1120 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1121 break;
1122 case BPF_JMP | BPF_JNE | BPF_K:
1123 case BPF_JMP | BPF_JEQ | BPF_K:
1124 case BPF_JMP | BPF_JGT | BPF_K:
1125 case BPF_JMP | BPF_JLT | BPF_K:
1126 case BPF_JMP | BPF_JGE | BPF_K:
1127 case BPF_JMP | BPF_JLE | BPF_K:
1128 /*
1129 * Need sign-extended load, so only positive
1130 * values can be used as imm in cmplwi
1131 */
1132 if (imm >= 0 && imm < 32768) {
1133 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1134 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1135 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1136 } else {
1137 /* sign-extending load ... but unsigned comparison */
1138 PPC_EX32(_R0, imm);
1139 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1140 PPC_LI32(_R0, imm);
1141 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1142 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1143 }
1144 break;
1145 case BPF_JMP32 | BPF_JNE | BPF_K:
1146 case BPF_JMP32 | BPF_JEQ | BPF_K:
1147 case BPF_JMP32 | BPF_JGT | BPF_K:
1148 case BPF_JMP32 | BPF_JLT | BPF_K:
1149 case BPF_JMP32 | BPF_JGE | BPF_K:
1150 case BPF_JMP32 | BPF_JLE | BPF_K:
1151 if (imm >= 0 && imm < 65536) {
1152 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1153 } else {
1154 PPC_LI32(_R0, imm);
1155 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1156 }
1157 break;
1158 }
1159 case BPF_JMP | BPF_JSGT | BPF_K:
1160 case BPF_JMP | BPF_JSLT | BPF_K:
1161 case BPF_JMP | BPF_JSGE | BPF_K:
1162 case BPF_JMP | BPF_JSLE | BPF_K:
1163 if (imm >= 0 && imm < 65536) {
1164 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1165 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1166 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1167 } else {
1168 /* sign-extending load */
1169 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1170 PPC_LI32(_R0, imm);
1171 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1172 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1173 }
1174 break;
1175 case BPF_JMP32 | BPF_JSGT | BPF_K:
1176 case BPF_JMP32 | BPF_JSLT | BPF_K:
1177 case BPF_JMP32 | BPF_JSGE | BPF_K:
1178 case BPF_JMP32 | BPF_JSLE | BPF_K:
1179 /*
1180 * signed comparison, so any 16-bit value
1181 * can be used in cmpwi
1182 */
1183 if (imm >= -32768 && imm < 32768) {
1184 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1185 } else {
1186 /* sign-extending load */
1187 PPC_LI32(_R0, imm);
1188 EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1189 }
1190 break;
1191 case BPF_JMP | BPF_JSET | BPF_K:
1192 /* andi does not sign-extend the immediate */
1193 if (imm >= 0 && imm < 32768) {
1194 /* PPC_ANDI is _only/always_ dot-form */
1195 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1196 } else {
1197 PPC_LI32(_R0, imm);
1198 if (imm < 0) {
1199 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1200 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1201 }
1202 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1203 }
1204 break;
1205 case BPF_JMP32 | BPF_JSET | BPF_K:
1206 /* andi does not sign-extend the immediate */
1207 if (imm >= 0 && imm < 32768) {
1208 /* PPC_ANDI is _only/always_ dot-form */
1209 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1210 } else {
1211 PPC_LI32(_R0, imm);
1212 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1213 }
1214 break;
1215 }
1216 PPC_BCC(true_cond, addrs[i + 1 + off]);
1217 break;
1218
1219 /*
1220 * Tail call
1221 */
1222 case BPF_JMP | BPF_TAIL_CALL:
1223 ctx->seen |= SEEN_TAILCALL;
1224 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1225 if (ret < 0)
1226 return ret;
1227 break;
1228
1229 default:
1230 /*
1231 * The filter contains something cruel & unusual.
1232 * We don't handle it, but also there shouldn't be
1233 * anything missing from our list.
1234 */
1235 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1236 return -EOPNOTSUPP;
1237 }
1238 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1239 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1240 EMIT(PPC_RAW_LI(dst_reg_h, 0));
1241 }
1242
1243 /* Set end-of-body-code address for exit. */
1244 addrs[i] = ctx->idx * 4;
1245
1246 return 0;
1247}