Linux Audio

Check our new training course

Loading...
Note: File does not exist in v4.17.
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * eBPF JIT compiler for PPC32
   4 *
   5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
   6 *		  CS GROUP France
   7 *
   8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
   9 */
  10#include <linux/moduleloader.h>
  11#include <asm/cacheflush.h>
  12#include <asm/asm-compat.h>
  13#include <linux/netdevice.h>
  14#include <linux/filter.h>
  15#include <linux/if_vlan.h>
  16#include <asm/kprobes.h>
  17#include <linux/bpf.h>
  18
  19#include "bpf_jit.h"
  20
  21/*
  22 * Stack layout:
  23 *
  24 *		[	prev sp		] <-------------
  25 *		[   nv gpr save area	] 16 * 4	|
  26 * fp (r31) -->	[   ebpf stack space	] upto 512	|
  27 *		[     frame header	] 16		|
  28 * sp (r1) --->	[    stack pointer	] --------------
  29 */
  30
  31/* for gpr non volatile registers r17 to r31 (14) + tail call */
  32#define BPF_PPC_STACK_SAVE	(15 * 4 + 4)
  33/* stack frame, ensure this is quadword aligned */
  34#define BPF_PPC_STACKFRAME(ctx)	(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
  35
  36#define PPC_EX32(r, i)		EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
  37
  38/* PPC NVR range -- update this if we ever use NVRs below r17 */
  39#define BPF_PPC_NVR_MIN		_R17
  40#define BPF_PPC_TC		_R16
  41
  42/* BPF register usage */
  43#define TMP_REG			(MAX_BPF_JIT_REG + 0)
  44
  45/* BPF to ppc register mappings */
  46void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
  47{
  48	/* function return value */
  49	ctx->b2p[BPF_REG_0] = _R12;
  50	/* function arguments */
  51	ctx->b2p[BPF_REG_1] = _R4;
  52	ctx->b2p[BPF_REG_2] = _R6;
  53	ctx->b2p[BPF_REG_3] = _R8;
  54	ctx->b2p[BPF_REG_4] = _R10;
  55	ctx->b2p[BPF_REG_5] = _R22;
  56	/* non volatile registers */
  57	ctx->b2p[BPF_REG_6] = _R24;
  58	ctx->b2p[BPF_REG_7] = _R26;
  59	ctx->b2p[BPF_REG_8] = _R28;
  60	ctx->b2p[BPF_REG_9] = _R30;
  61	/* frame pointer aka BPF_REG_10 */
  62	ctx->b2p[BPF_REG_FP] = _R18;
  63	/* eBPF jit internal registers */
  64	ctx->b2p[BPF_REG_AX] = _R20;
  65	ctx->b2p[TMP_REG] = _R31;		/* 32 bits */
  66}
  67
  68static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
  69{
  70	if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
  71		return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
  72
  73	WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
  74	/* Use the hole we have left for alignment */
  75	return BPF_PPC_STACKFRAME(ctx) - 4;
  76}
  77
  78#define SEEN_VREG_MASK		0x1ff80000 /* Volatile registers r3-r12 */
  79#define SEEN_NVREG_FULL_MASK	0x0003ffff /* Non volatile registers r14-r31 */
  80#define SEEN_NVREG_TEMP_MASK	0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */
  81
  82void bpf_jit_realloc_regs(struct codegen_context *ctx)
  83{
  84	unsigned int nvreg_mask;
  85
  86	if (ctx->seen & SEEN_FUNC)
  87		nvreg_mask = SEEN_NVREG_TEMP_MASK;
  88	else
  89		nvreg_mask = SEEN_NVREG_FULL_MASK;
  90
  91	while (ctx->seen & nvreg_mask &&
  92	      (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
  93		int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab));
  94		int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
  95		int i;
  96
  97		for (i = BPF_REG_0; i <= TMP_REG; i++) {
  98			if (ctx->b2p[i] != old)
  99				continue;
 100			ctx->b2p[i] = new;
 101			bpf_set_seen_register(ctx, new);
 102			bpf_clear_seen_register(ctx, old);
 103			if (i != TMP_REG) {
 104				bpf_set_seen_register(ctx, new - 1);
 105				bpf_clear_seen_register(ctx, old - 1);
 106			}
 107			break;
 108		}
 109	}
 110}
 111
 112void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 113{
 114	int i;
 115
 116	/* Initialize tail_call_cnt, to be skipped if we do tail calls. */
 117	EMIT(PPC_RAW_LI(_R4, 0));
 118
 119#define BPF_TAILCALL_PROLOGUE_SIZE	4
 120
 121	EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
 122
 123	if (ctx->seen & SEEN_TAILCALL)
 124		EMIT(PPC_RAW_STW(_R4, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
 125
 126	/* First arg comes in as a 32 bits pointer. */
 127	EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3));
 128	EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0));
 129
 130	/*
 131	 * We need a stack frame, but we don't necessarily need to
 132	 * save/restore LR unless we call other functions
 133	 */
 134	if (ctx->seen & SEEN_FUNC)
 135		EMIT(PPC_RAW_MFLR(_R0));
 136
 137	/*
 138	 * Back up non-volatile regs -- registers r18-r31
 139	 */
 140	for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
 141		if (bpf_is_seen_register(ctx, i))
 142			EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
 143
 144	/* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
 145	if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
 146		EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8);
 147		EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12);
 148	}
 149
 150	/* Setup frame pointer to point to the bpf stack area */
 151	if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) {
 152		EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0));
 153		EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
 154				  STACK_FRAME_MIN_SIZE + ctx->stack_size));
 155	}
 156
 157	if (ctx->seen & SEEN_FUNC)
 158		EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
 159}
 160
 161static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
 162{
 163	int i;
 164
 165	/* Restore NVRs */
 166	for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
 167		if (bpf_is_seen_register(ctx, i))
 168			EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
 169
 170	if (ctx->seen & SEEN_FUNC)
 171		EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
 172
 173	/* Tear down our stack frame */
 174	EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
 175
 176	if (ctx->seen & SEEN_FUNC)
 177		EMIT(PPC_RAW_MTLR(_R0));
 178
 179}
 180
 181void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
 182{
 183	EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
 184
 185	bpf_jit_emit_common_epilogue(image, ctx);
 186
 187	EMIT(PPC_RAW_BLR());
 188}
 189
 190int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
 191{
 192	s32 rel = (s32)func - (s32)(image + ctx->idx);
 193
 194	if (image && rel < 0x2000000 && rel >= -0x2000000) {
 195		PPC_BL(func);
 196		EMIT(PPC_RAW_NOP());
 197		EMIT(PPC_RAW_NOP());
 198		EMIT(PPC_RAW_NOP());
 199	} else {
 200		/* Load function address into r0 */
 201		EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
 202		EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
 203		EMIT(PPC_RAW_MTCTR(_R0));
 204		EMIT(PPC_RAW_BCTRL());
 205	}
 206
 207	return 0;
 208}
 209
 210static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
 211{
 212	/*
 213	 * By now, the eBPF program has already setup parameters in r3-r6
 214	 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
 215	 * r5-r6/BPF_REG_2 - pointer to bpf_array
 216	 * r7-r8/BPF_REG_3 - index in bpf_array
 217	 */
 218	int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
 219	int b2p_index = bpf_to_ppc(BPF_REG_3);
 220
 221	/*
 222	 * if (index >= array->map.max_entries)
 223	 *   goto out;
 224	 */
 225	EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
 226	EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
 227	EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
 228	PPC_BCC_SHORT(COND_GE, out);
 229
 230	/*
 231	 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
 232	 *   goto out;
 233	 */
 234	EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
 235	/* tail_call_cnt++; */
 236	EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
 237	PPC_BCC_SHORT(COND_GE, out);
 238
 239	/* prog = array->ptrs[index]; */
 240	EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
 241	EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
 242	EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
 243
 244	/*
 245	 * if (prog == NULL)
 246	 *   goto out;
 247	 */
 248	EMIT(PPC_RAW_CMPLWI(_R3, 0));
 249	PPC_BCC_SHORT(COND_EQ, out);
 250
 251	/* goto *(prog->bpf_func + prologue_size); */
 252	EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
 253	EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
 254	EMIT(PPC_RAW_MTCTR(_R3));
 255
 256	EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1)));
 257
 258	/* Put tail_call_cnt in r4 */
 259	EMIT(PPC_RAW_MR(_R4, _R0));
 260
 261	/* tear restore NVRs, ... */
 262	bpf_jit_emit_common_epilogue(image, ctx);
 263
 264	EMIT(PPC_RAW_BCTR());
 265
 266	/* out: */
 267	return 0;
 268}
 269
 270/* Assemble the body code between the prologue & epilogue */
 271int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
 272		       u32 *addrs, int pass)
 273{
 274	const struct bpf_insn *insn = fp->insnsi;
 275	int flen = fp->len;
 276	int i, ret;
 277
 278	/* Start of epilogue code - will only be valid 2nd pass onwards */
 279	u32 exit_addr = addrs[flen];
 280
 281	for (i = 0; i < flen; i++) {
 282		u32 code = insn[i].code;
 283		u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
 284		u32 dst_reg_h = dst_reg - 1;
 285		u32 src_reg = bpf_to_ppc(insn[i].src_reg);
 286		u32 src_reg_h = src_reg - 1;
 287		u32 ax_reg = bpf_to_ppc(BPF_REG_AX);
 288		u32 tmp_reg = bpf_to_ppc(TMP_REG);
 289		u32 size = BPF_SIZE(code);
 290		u32 save_reg, ret_reg;
 291		s16 off = insn[i].off;
 292		s32 imm = insn[i].imm;
 293		bool func_addr_fixed;
 294		u64 func_addr;
 295		u32 true_cond;
 296		u32 tmp_idx;
 297		int j;
 298
 299		/*
 300		 * addrs[] maps a BPF bytecode address into a real offset from
 301		 * the start of the body code.
 302		 */
 303		addrs[i] = ctx->idx * 4;
 304
 305		/*
 306		 * As an optimization, we note down which registers
 307		 * are used so that we can only save/restore those in our
 308		 * prologue and epilogue. We do this here regardless of whether
 309		 * the actual BPF instruction uses src/dst registers or not
 310		 * (for instance, BPF_CALL does not use them). The expectation
 311		 * is that those instructions will have src_reg/dst_reg set to
 312		 * 0. Even otherwise, we just lose some prologue/epilogue
 313		 * optimization but everything else should work without
 314		 * any issues.
 315		 */
 316		if (dst_reg >= 3 && dst_reg < 32) {
 317			bpf_set_seen_register(ctx, dst_reg);
 318			bpf_set_seen_register(ctx, dst_reg_h);
 319		}
 320
 321		if (src_reg >= 3 && src_reg < 32) {
 322			bpf_set_seen_register(ctx, src_reg);
 323			bpf_set_seen_register(ctx, src_reg_h);
 324		}
 325
 326		switch (code) {
 327		/*
 328		 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
 329		 */
 330		case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
 331			EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
 332			break;
 333		case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
 334			EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
 335			EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
 336			break;
 337		case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
 338			EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
 339			break;
 340		case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
 341			EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
 342			EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
 343			break;
 344		case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
 345			imm = -imm;
 346			fallthrough;
 347		case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
 348			if (IMM_HA(imm) & 0xffff)
 349				EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
 350			if (IMM_L(imm))
 351				EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
 352			break;
 353		case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
 354			imm = -imm;
 355			fallthrough;
 356		case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
 357			if (!imm)
 358				break;
 359
 360			if (imm >= -32768 && imm < 32768) {
 361				EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
 362			} else {
 363				PPC_LI32(_R0, imm);
 364				EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0));
 365			}
 366			if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
 367				EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
 368			else
 369				EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
 370			break;
 371		case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
 372			bpf_set_seen_register(ctx, tmp_reg);
 373			EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h));
 374			EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
 375			EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
 376			EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
 377			EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
 378			EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
 379			break;
 380		case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
 381			EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
 382			break;
 383		case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
 384			if (imm >= -32768 && imm < 32768) {
 385				EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
 386			} else {
 387				PPC_LI32(_R0, imm);
 388				EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0));
 389			}
 390			break;
 391		case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
 392			if (!imm) {
 393				PPC_LI32(dst_reg, 0);
 394				PPC_LI32(dst_reg_h, 0);
 395				break;
 396			}
 397			if (imm == 1)
 398				break;
 399			if (imm == -1) {
 400				EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
 401				EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
 402				break;
 403			}
 404			bpf_set_seen_register(ctx, tmp_reg);
 405			PPC_LI32(tmp_reg, imm);
 406			EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
 407			if (imm < 0)
 408				EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
 409			EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg));
 410			EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
 411			EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
 412			break;
 413		case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
 414			EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
 415			break;
 416		case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
 417			EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg));
 418			EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
 419			EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
 420			break;
 421		case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
 422			return -EOPNOTSUPP;
 423		case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
 424			return -EOPNOTSUPP;
 425		case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
 426			if (!imm)
 427				return -EINVAL;
 428			if (imm == 1)
 429				break;
 430
 431			PPC_LI32(_R0, imm);
 432			EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0));
 433			break;
 434		case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
 435			if (!imm)
 436				return -EINVAL;
 437
 438			if (!is_power_of_2((u32)imm)) {
 439				bpf_set_seen_register(ctx, tmp_reg);
 440				PPC_LI32(tmp_reg, imm);
 441				EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg));
 442				EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
 443				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
 444				break;
 445			}
 446			if (imm == 1)
 447				EMIT(PPC_RAW_LI(dst_reg, 0));
 448			else
 449				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
 450
 451			break;
 452		case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
 453			if (!imm)
 454				return -EINVAL;
 455			if (imm < 0)
 456				imm = -imm;
 457			if (!is_power_of_2(imm))
 458				return -EOPNOTSUPP;
 459			if (imm == 1)
 460				EMIT(PPC_RAW_LI(dst_reg, 0));
 461			else
 462				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
 463			EMIT(PPC_RAW_LI(dst_reg_h, 0));
 464			break;
 465		case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
 466			if (!imm)
 467				return -EINVAL;
 468			if (!is_power_of_2(abs(imm)))
 469				return -EOPNOTSUPP;
 470
 471			if (imm < 0) {
 472				EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
 473				EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
 474				imm = -imm;
 475			}
 476			if (imm == 1)
 477				break;
 478			imm = ilog2(imm);
 479			EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
 480			EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
 481			EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
 482			break;
 483		case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
 484			EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
 485			break;
 486		case BPF_ALU64 | BPF_NEG: /* dst = -dst */
 487			EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
 488			EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
 489			break;
 490
 491		/*
 492		 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
 493		 */
 494		case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
 495			EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
 496			EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
 497			break;
 498		case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
 499			EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
 500			break;
 501		case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
 502			if (imm >= 0)
 503				EMIT(PPC_RAW_LI(dst_reg_h, 0));
 504			fallthrough;
 505		case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
 506			if (!IMM_H(imm)) {
 507				EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
 508			} else if (!IMM_L(imm)) {
 509				EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
 510			} else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
 511				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
 512						    32 - fls(imm), 32 - ffs(imm)));
 513			} else {
 514				PPC_LI32(_R0, imm);
 515				EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0));
 516			}
 517			break;
 518		case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
 519			EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
 520			EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
 521			break;
 522		case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
 523			EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
 524			break;
 525		case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
 526			/* Sign-extended */
 527			if (imm < 0)
 528				EMIT(PPC_RAW_LI(dst_reg_h, -1));
 529			fallthrough;
 530		case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
 531			if (IMM_L(imm))
 532				EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
 533			if (IMM_H(imm))
 534				EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
 535			break;
 536		case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
 537			if (dst_reg == src_reg) {
 538				EMIT(PPC_RAW_LI(dst_reg, 0));
 539				EMIT(PPC_RAW_LI(dst_reg_h, 0));
 540			} else {
 541				EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
 542				EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
 543			}
 544			break;
 545		case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
 546			if (dst_reg == src_reg)
 547				EMIT(PPC_RAW_LI(dst_reg, 0));
 548			else
 549				EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
 550			break;
 551		case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
 552			if (imm < 0)
 553				EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
 554			fallthrough;
 555		case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
 556			if (IMM_L(imm))
 557				EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
 558			if (IMM_H(imm))
 559				EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
 560			break;
 561		case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
 562			EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
 563			break;
 564		case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
 565			bpf_set_seen_register(ctx, tmp_reg);
 566			EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
 567			EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
 568			EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
 569			EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0));
 570			EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
 571			EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
 572			EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
 573			EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
 574			break;
 575		case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */
 576			if (!imm)
 577				break;
 578			EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
 579			break;
 580		case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */
 581			if (imm < 0)
 582				return -EINVAL;
 583			if (!imm)
 584				break;
 585			if (imm < 32) {
 586				EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
 587				EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
 588				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
 589				break;
 590			}
 591			if (imm < 64)
 592				EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
 593			else
 594				EMIT(PPC_RAW_LI(dst_reg_h, 0));
 595			EMIT(PPC_RAW_LI(dst_reg, 0));
 596			break;
 597		case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
 598			EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
 599			break;
 600		case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
 601			bpf_set_seen_register(ctx, tmp_reg);
 602			EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
 603			EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
 604			EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
 605			EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
 606			EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
 607			EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
 608			EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
 609			EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
 610			break;
 611		case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
 612			if (!imm)
 613				break;
 614			EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
 615			break;
 616		case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
 617			if (imm < 0)
 618				return -EINVAL;
 619			if (!imm)
 620				break;
 621			if (imm < 32) {
 622				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
 623				EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
 624				EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
 625				break;
 626			}
 627			if (imm < 64)
 628				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
 629			else
 630				EMIT(PPC_RAW_LI(dst_reg, 0));
 631			EMIT(PPC_RAW_LI(dst_reg_h, 0));
 632			break;
 633		case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
 634			EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
 635			break;
 636		case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
 637			bpf_set_seen_register(ctx, tmp_reg);
 638			EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
 639			EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
 640			EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
 641			EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
 642			EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
 643			EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
 644			EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
 645			EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
 646			EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
 647			EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
 648			break;
 649		case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
 650			if (!imm)
 651				break;
 652			EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
 653			break;
 654		case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
 655			if (imm < 0)
 656				return -EINVAL;
 657			if (!imm)
 658				break;
 659			if (imm < 32) {
 660				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
 661				EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
 662				EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
 663				break;
 664			}
 665			if (imm < 64)
 666				EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
 667			else
 668				EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
 669			EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
 670			break;
 671
 672		/*
 673		 * MOV
 674		 */
 675		case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
 676			if (dst_reg == src_reg)
 677				break;
 678			EMIT(PPC_RAW_MR(dst_reg, src_reg));
 679			EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
 680			break;
 681		case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
 682			/* special mov32 for zext */
 683			if (imm == 1)
 684				EMIT(PPC_RAW_LI(dst_reg_h, 0));
 685			else if (dst_reg != src_reg)
 686				EMIT(PPC_RAW_MR(dst_reg, src_reg));
 687			break;
 688		case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
 689			PPC_LI32(dst_reg, imm);
 690			PPC_EX32(dst_reg_h, imm);
 691			break;
 692		case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
 693			PPC_LI32(dst_reg, imm);
 694			break;
 695
 696		/*
 697		 * BPF_FROM_BE/LE
 698		 */
 699		case BPF_ALU | BPF_END | BPF_FROM_LE:
 700			switch (imm) {
 701			case 16:
 702				/* Copy 16 bits to upper part */
 703				EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
 704				/* Rotate 8 bits right & mask */
 705				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
 706				break;
 707			case 32:
 708				/*
 709				 * Rotate word left by 8 bits:
 710				 * 2 bytes are already in their final position
 711				 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
 712				 */
 713				EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31));
 714				/* Rotate 24 bits and insert byte 1 */
 715				EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7));
 716				/* Rotate 24 bits and insert byte 3 */
 717				EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23));
 718				EMIT(PPC_RAW_MR(dst_reg, _R0));
 719				break;
 720			case 64:
 721				bpf_set_seen_register(ctx, tmp_reg);
 722				EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
 723				EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31));
 724				/* Rotate 24 bits and insert byte 1 */
 725				EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
 726				EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7));
 727				/* Rotate 24 bits and insert byte 3 */
 728				EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
 729				EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23));
 730				EMIT(PPC_RAW_MR(dst_reg, _R0));
 731				EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
 732				break;
 733			}
 734			break;
 735		case BPF_ALU | BPF_END | BPF_FROM_BE:
 736			switch (imm) {
 737			case 16:
 738				/* zero-extend 16 bits into 32 bits */
 739				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
 740				break;
 741			case 32:
 742			case 64:
 743				/* nop */
 744				break;
 745			}
 746			break;
 747
 748		/*
 749		 * BPF_ST NOSPEC (speculation barrier)
 750		 */
 751		case BPF_ST | BPF_NOSPEC:
 752			break;
 753
 754		/*
 755		 * BPF_ST(X)
 756		 */
 757		case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
 758			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
 759			break;
 760		case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
 761			PPC_LI32(_R0, imm);
 762			EMIT(PPC_RAW_STB(_R0, dst_reg, off));
 763			break;
 764		case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
 765			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
 766			break;
 767		case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
 768			PPC_LI32(_R0, imm);
 769			EMIT(PPC_RAW_STH(_R0, dst_reg, off));
 770			break;
 771		case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
 772			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
 773			break;
 774		case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
 775			PPC_LI32(_R0, imm);
 776			EMIT(PPC_RAW_STW(_R0, dst_reg, off));
 777			break;
 778		case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
 779			EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
 780			EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
 781			break;
 782		case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
 783			PPC_LI32(_R0, imm);
 784			EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
 785			PPC_EX32(_R0, imm);
 786			EMIT(PPC_RAW_STW(_R0, dst_reg, off));
 787			break;
 788
 789		/*
 790		 * BPF_STX ATOMIC (atomic ops)
 791		 */
 792		case BPF_STX | BPF_ATOMIC | BPF_W:
 793			save_reg = _R0;
 794			ret_reg = src_reg;
 795
 796			bpf_set_seen_register(ctx, tmp_reg);
 797			bpf_set_seen_register(ctx, ax_reg);
 798
 799			/* Get offset into TMP_REG */
 800			EMIT(PPC_RAW_LI(tmp_reg, off));
 801			tmp_idx = ctx->idx * 4;
 802			/* load value from memory into r0 */
 803			EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
 804
 805			/* Save old value in BPF_REG_AX */
 806			if (imm & BPF_FETCH)
 807				EMIT(PPC_RAW_MR(ax_reg, _R0));
 808
 809			switch (imm) {
 810			case BPF_ADD:
 811			case BPF_ADD | BPF_FETCH:
 812				EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
 813				break;
 814			case BPF_AND:
 815			case BPF_AND | BPF_FETCH:
 816				EMIT(PPC_RAW_AND(_R0, _R0, src_reg));
 817				break;
 818			case BPF_OR:
 819			case BPF_OR | BPF_FETCH:
 820				EMIT(PPC_RAW_OR(_R0, _R0, src_reg));
 821				break;
 822			case BPF_XOR:
 823			case BPF_XOR | BPF_FETCH:
 824				EMIT(PPC_RAW_XOR(_R0, _R0, src_reg));
 825				break;
 826			case BPF_CMPXCHG:
 827				/*
 828				 * Return old value in BPF_REG_0 for BPF_CMPXCHG &
 829				 * in src_reg for other cases.
 830				 */
 831				ret_reg = bpf_to_ppc(BPF_REG_0);
 832
 833				/* Compare with old value in BPF_REG_0 */
 834				EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0));
 835				/* Don't set if different from old value */
 836				PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
 837				fallthrough;
 838			case BPF_XCHG:
 839				save_reg = src_reg;
 840				break;
 841			default:
 842				pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
 843						   code, i);
 844				return -EOPNOTSUPP;
 845			}
 846
 847			/* store new value */
 848			EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg));
 849			/* we're done if this succeeded */
 850			PPC_BCC_SHORT(COND_NE, tmp_idx);
 851
 852			/* For the BPF_FETCH variant, get old data into src_reg */
 853			if (imm & BPF_FETCH) {
 854				EMIT(PPC_RAW_MR(ret_reg, ax_reg));
 855				if (!fp->aux->verifier_zext)
 856					EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */
 857			}
 858			break;
 859
 860		case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */
 861			return -EOPNOTSUPP;
 862
 863		/*
 864		 * BPF_LDX
 865		 */
 866		case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
 867		case BPF_LDX | BPF_PROBE_MEM | BPF_B:
 868		case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
 869		case BPF_LDX | BPF_PROBE_MEM | BPF_H:
 870		case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
 871		case BPF_LDX | BPF_PROBE_MEM | BPF_W:
 872		case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
 873		case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
 874			/*
 875			 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
 876			 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
 877			 * load only if addr is kernel address (see is_kernel_addr()), otherwise
 878			 * set dst_reg=0 and move on.
 879			 */
 880			if (BPF_MODE(code) == BPF_PROBE_MEM) {
 881				PPC_LI32(_R0, TASK_SIZE - off);
 882				EMIT(PPC_RAW_CMPLW(src_reg, _R0));
 883				PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4);
 884				EMIT(PPC_RAW_LI(dst_reg, 0));
 885				/*
 886				 * For BPF_DW case, "li reg_h,0" would be needed when
 887				 * !fp->aux->verifier_zext. Emit NOP otherwise.
 888				 *
 889				 * Note that "li reg_h,0" is emitted for BPF_B/H/W case,
 890				 * if necessary. So, jump there insted of emitting an
 891				 * additional "li reg_h,0" instruction.
 892				 */
 893				if (size == BPF_DW && !fp->aux->verifier_zext)
 894					EMIT(PPC_RAW_LI(dst_reg_h, 0));
 895				else
 896					EMIT(PPC_RAW_NOP());
 897				/*
 898				 * Need to jump two instructions instead of one for BPF_DW case
 899				 * as there are two load instructions for dst_reg_h & dst_reg
 900				 * respectively.
 901				 */
 902				if (size == BPF_DW)
 903					PPC_JMP((ctx->idx + 3) * 4);
 904				else
 905					PPC_JMP((ctx->idx + 2) * 4);
 906			}
 907
 908			switch (size) {
 909			case BPF_B:
 910				EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
 911				break;
 912			case BPF_H:
 913				EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
 914				break;
 915			case BPF_W:
 916				EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
 917				break;
 918			case BPF_DW:
 919				EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
 920				EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
 921				break;
 922			}
 923
 924			if (size != BPF_DW && !fp->aux->verifier_zext)
 925				EMIT(PPC_RAW_LI(dst_reg_h, 0));
 926
 927			if (BPF_MODE(code) == BPF_PROBE_MEM) {
 928				int insn_idx = ctx->idx - 1;
 929				int jmp_off = 4;
 930
 931				/*
 932				 * In case of BPF_DW, two lwz instructions are emitted, one
 933				 * for higher 32-bit and another for lower 32-bit. So, set
 934				 * ex->insn to the first of the two and jump over both
 935				 * instructions in fixup.
 936				 *
 937				 * Similarly, with !verifier_zext, two instructions are
 938				 * emitted for BPF_B/H/W case. So, set ex->insn to the
 939				 * instruction that could fault and skip over both
 940				 * instructions.
 941				 */
 942				if (size == BPF_DW || !fp->aux->verifier_zext) {
 943					insn_idx -= 1;
 944					jmp_off += 4;
 945				}
 946
 947				ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
 948							    jmp_off, dst_reg);
 949				if (ret)
 950					return ret;
 951			}
 952			break;
 953
 954		/*
 955		 * Doubleword load
 956		 * 16 byte instruction that uses two 'struct bpf_insn'
 957		 */
 958		case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
 959			tmp_idx = ctx->idx;
 960			PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
 961			PPC_LI32(dst_reg, (u32)insn[i].imm);
 962			/* padding to allow full 4 instructions for later patching */
 963			for (j = ctx->idx - tmp_idx; j < 4; j++)
 964				EMIT(PPC_RAW_NOP());
 965			/* Adjust for two bpf instructions */
 966			addrs[++i] = ctx->idx * 4;
 967			break;
 968
 969		/*
 970		 * Return/Exit
 971		 */
 972		case BPF_JMP | BPF_EXIT:
 973			/*
 974			 * If this isn't the very last instruction, branch to
 975			 * the epilogue. If we _are_ the last instruction,
 976			 * we'll just fall through to the epilogue.
 977			 */
 978			if (i != flen - 1) {
 979				ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
 980				if (ret)
 981					return ret;
 982			}
 983			/* else fall through to the epilogue */
 984			break;
 985
 986		/*
 987		 * Call kernel helper or bpf function
 988		 */
 989		case BPF_JMP | BPF_CALL:
 990			ctx->seen |= SEEN_FUNC;
 991
 992			ret = bpf_jit_get_func_addr(fp, &insn[i], false,
 993						    &func_addr, &func_addr_fixed);
 994			if (ret < 0)
 995				return ret;
 996
 997			if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
 998				EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8));
 999				EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
1000			}
1001
1002			ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1003			if (ret)
1004				return ret;
1005
1006			EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3));
1007			EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4));
1008			break;
1009
1010		/*
1011		 * Jumps and branches
1012		 */
1013		case BPF_JMP | BPF_JA:
1014			PPC_JMP(addrs[i + 1 + off]);
1015			break;
1016
1017		case BPF_JMP | BPF_JGT | BPF_K:
1018		case BPF_JMP | BPF_JGT | BPF_X:
1019		case BPF_JMP | BPF_JSGT | BPF_K:
1020		case BPF_JMP | BPF_JSGT | BPF_X:
1021		case BPF_JMP32 | BPF_JGT | BPF_K:
1022		case BPF_JMP32 | BPF_JGT | BPF_X:
1023		case BPF_JMP32 | BPF_JSGT | BPF_K:
1024		case BPF_JMP32 | BPF_JSGT | BPF_X:
1025			true_cond = COND_GT;
1026			goto cond_branch;
1027		case BPF_JMP | BPF_JLT | BPF_K:
1028		case BPF_JMP | BPF_JLT | BPF_X:
1029		case BPF_JMP | BPF_JSLT | BPF_K:
1030		case BPF_JMP | BPF_JSLT | BPF_X:
1031		case BPF_JMP32 | BPF_JLT | BPF_K:
1032		case BPF_JMP32 | BPF_JLT | BPF_X:
1033		case BPF_JMP32 | BPF_JSLT | BPF_K:
1034		case BPF_JMP32 | BPF_JSLT | BPF_X:
1035			true_cond = COND_LT;
1036			goto cond_branch;
1037		case BPF_JMP | BPF_JGE | BPF_K:
1038		case BPF_JMP | BPF_JGE | BPF_X:
1039		case BPF_JMP | BPF_JSGE | BPF_K:
1040		case BPF_JMP | BPF_JSGE | BPF_X:
1041		case BPF_JMP32 | BPF_JGE | BPF_K:
1042		case BPF_JMP32 | BPF_JGE | BPF_X:
1043		case BPF_JMP32 | BPF_JSGE | BPF_K:
1044		case BPF_JMP32 | BPF_JSGE | BPF_X:
1045			true_cond = COND_GE;
1046			goto cond_branch;
1047		case BPF_JMP | BPF_JLE | BPF_K:
1048		case BPF_JMP | BPF_JLE | BPF_X:
1049		case BPF_JMP | BPF_JSLE | BPF_K:
1050		case BPF_JMP | BPF_JSLE | BPF_X:
1051		case BPF_JMP32 | BPF_JLE | BPF_K:
1052		case BPF_JMP32 | BPF_JLE | BPF_X:
1053		case BPF_JMP32 | BPF_JSLE | BPF_K:
1054		case BPF_JMP32 | BPF_JSLE | BPF_X:
1055			true_cond = COND_LE;
1056			goto cond_branch;
1057		case BPF_JMP | BPF_JEQ | BPF_K:
1058		case BPF_JMP | BPF_JEQ | BPF_X:
1059		case BPF_JMP32 | BPF_JEQ | BPF_K:
1060		case BPF_JMP32 | BPF_JEQ | BPF_X:
1061			true_cond = COND_EQ;
1062			goto cond_branch;
1063		case BPF_JMP | BPF_JNE | BPF_K:
1064		case BPF_JMP | BPF_JNE | BPF_X:
1065		case BPF_JMP32 | BPF_JNE | BPF_K:
1066		case BPF_JMP32 | BPF_JNE | BPF_X:
1067			true_cond = COND_NE;
1068			goto cond_branch;
1069		case BPF_JMP | BPF_JSET | BPF_K:
1070		case BPF_JMP | BPF_JSET | BPF_X:
1071		case BPF_JMP32 | BPF_JSET | BPF_K:
1072		case BPF_JMP32 | BPF_JSET | BPF_X:
1073			true_cond = COND_NE;
1074			/* fallthrough; */
1075
1076cond_branch:
1077			switch (code) {
1078			case BPF_JMP | BPF_JGT | BPF_X:
1079			case BPF_JMP | BPF_JLT | BPF_X:
1080			case BPF_JMP | BPF_JGE | BPF_X:
1081			case BPF_JMP | BPF_JLE | BPF_X:
1082			case BPF_JMP | BPF_JEQ | BPF_X:
1083			case BPF_JMP | BPF_JNE | BPF_X:
1084				/* unsigned comparison */
1085				EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1086				PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1087				EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1088				break;
1089			case BPF_JMP32 | BPF_JGT | BPF_X:
1090			case BPF_JMP32 | BPF_JLT | BPF_X:
1091			case BPF_JMP32 | BPF_JGE | BPF_X:
1092			case BPF_JMP32 | BPF_JLE | BPF_X:
1093			case BPF_JMP32 | BPF_JEQ | BPF_X:
1094			case BPF_JMP32 | BPF_JNE | BPF_X:
1095				/* unsigned comparison */
1096				EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1097				break;
1098			case BPF_JMP | BPF_JSGT | BPF_X:
1099			case BPF_JMP | BPF_JSLT | BPF_X:
1100			case BPF_JMP | BPF_JSGE | BPF_X:
1101			case BPF_JMP | BPF_JSLE | BPF_X:
1102				/* signed comparison */
1103				EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1104				PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1105				EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1106				break;
1107			case BPF_JMP32 | BPF_JSGT | BPF_X:
1108			case BPF_JMP32 | BPF_JSLT | BPF_X:
1109			case BPF_JMP32 | BPF_JSGE | BPF_X:
1110			case BPF_JMP32 | BPF_JSLE | BPF_X:
1111				/* signed comparison */
1112				EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1113				break;
1114			case BPF_JMP | BPF_JSET | BPF_X:
1115				EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1116				PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1117				EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1118				break;
1119			case BPF_JMP32 | BPF_JSET | BPF_X: {
1120				EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1121				break;
1122			case BPF_JMP | BPF_JNE | BPF_K:
1123			case BPF_JMP | BPF_JEQ | BPF_K:
1124			case BPF_JMP | BPF_JGT | BPF_K:
1125			case BPF_JMP | BPF_JLT | BPF_K:
1126			case BPF_JMP | BPF_JGE | BPF_K:
1127			case BPF_JMP | BPF_JLE | BPF_K:
1128				/*
1129				 * Need sign-extended load, so only positive
1130				 * values can be used as imm in cmplwi
1131				 */
1132				if (imm >= 0 && imm < 32768) {
1133					EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1134					PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1135					EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1136				} else {
1137					/* sign-extending load ... but unsigned comparison */
1138					PPC_EX32(_R0, imm);
1139					EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1140					PPC_LI32(_R0, imm);
1141					PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1142					EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1143				}
1144				break;
1145			case BPF_JMP32 | BPF_JNE | BPF_K:
1146			case BPF_JMP32 | BPF_JEQ | BPF_K:
1147			case BPF_JMP32 | BPF_JGT | BPF_K:
1148			case BPF_JMP32 | BPF_JLT | BPF_K:
1149			case BPF_JMP32 | BPF_JGE | BPF_K:
1150			case BPF_JMP32 | BPF_JLE | BPF_K:
1151				if (imm >= 0 && imm < 65536) {
1152					EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1153				} else {
1154					PPC_LI32(_R0, imm);
1155					EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1156				}
1157				break;
1158			}
1159			case BPF_JMP | BPF_JSGT | BPF_K:
1160			case BPF_JMP | BPF_JSLT | BPF_K:
1161			case BPF_JMP | BPF_JSGE | BPF_K:
1162			case BPF_JMP | BPF_JSLE | BPF_K:
1163				if (imm >= 0 && imm < 65536) {
1164					EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1165					PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1166					EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1167				} else {
1168					/* sign-extending load */
1169					EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1170					PPC_LI32(_R0, imm);
1171					PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1172					EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1173				}
1174				break;
1175			case BPF_JMP32 | BPF_JSGT | BPF_K:
1176			case BPF_JMP32 | BPF_JSLT | BPF_K:
1177			case BPF_JMP32 | BPF_JSGE | BPF_K:
1178			case BPF_JMP32 | BPF_JSLE | BPF_K:
1179				/*
1180				 * signed comparison, so any 16-bit value
1181				 * can be used in cmpwi
1182				 */
1183				if (imm >= -32768 && imm < 32768) {
1184					EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1185				} else {
1186					/* sign-extending load */
1187					PPC_LI32(_R0, imm);
1188					EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1189				}
1190				break;
1191			case BPF_JMP | BPF_JSET | BPF_K:
1192				/* andi does not sign-extend the immediate */
1193				if (imm >= 0 && imm < 32768) {
1194					/* PPC_ANDI is _only/always_ dot-form */
1195					EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1196				} else {
1197					PPC_LI32(_R0, imm);
1198					if (imm < 0) {
1199						EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1200						PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1201					}
1202					EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1203				}
1204				break;
1205			case BPF_JMP32 | BPF_JSET | BPF_K:
1206				/* andi does not sign-extend the immediate */
1207				if (imm >= 0 && imm < 32768) {
1208					/* PPC_ANDI is _only/always_ dot-form */
1209					EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1210				} else {
1211					PPC_LI32(_R0, imm);
1212					EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1213				}
1214				break;
1215			}
1216			PPC_BCC(true_cond, addrs[i + 1 + off]);
1217			break;
1218
1219		/*
1220		 * Tail call
1221		 */
1222		case BPF_JMP | BPF_TAIL_CALL:
1223			ctx->seen |= SEEN_TAILCALL;
1224			ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1225			if (ret < 0)
1226				return ret;
1227			break;
1228
1229		default:
1230			/*
1231			 * The filter contains something cruel & unusual.
1232			 * We don't handle it, but also there shouldn't be
1233			 * anything missing from our list.
1234			 */
1235			pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1236			return -EOPNOTSUPP;
1237		}
1238		if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1239		    !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1240			EMIT(PPC_RAW_LI(dst_reg_h, 0));
1241	}
1242
1243	/* Set end-of-body-code address for exit. */
1244	addrs[i] = ctx->idx * 4;
1245
1246	return 0;
1247}