1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/moduleloader.h>
   3#include <linux/workqueue.h>
   4#include <linux/netdevice.h>
   5#include <linux/filter.h>
   6#include <linux/bpf.h>
   7#include <linux/cache.h>
   8#include <linux/if_vlan.h>
   9
  10#include <asm/cacheflush.h>
  11#include <asm/ptrace.h>
  12
  13#include "bpf_jit_64.h"
  14
  15static inline bool is_simm13(unsigned int value)
  16{
  17	return value + 0x1000 < 0x2000;
  18}
  19
  20static inline bool is_simm10(unsigned int value)
  21{
  22	return value + 0x200 < 0x400;
  23}
  24
  25static inline bool is_simm5(unsigned int value)
  26{
  27	return value + 0x10 < 0x20;
  28}
  29
  30static inline bool is_sethi(unsigned int value)
  31{
  32	return (value & ~0x3fffff) == 0;
  33}
  34
  35static void bpf_flush_icache(void *start_, void *end_)
  36{
  37	/* Cheetah's I-cache is fully coherent.  */
  38	if (tlb_type == spitfire) {
  39		unsigned long start = (unsigned long) start_;
  40		unsigned long end = (unsigned long) end_;
  41
  42		start &= ~7UL;
  43		end = (end + 7UL) & ~7UL;
  44		while (start < end) {
  45			flushi(start);
  46			start += 32;
  47		}
  48	}
  49}
  50
  51#define S13(X)		((X) & 0x1fff)
  52#define S5(X)		((X) & 0x1f)
  53#define IMMED		0x00002000
  54#define RD(X)		((X) << 25)
  55#define RS1(X)		((X) << 14)
  56#define RS2(X)		((X))
  57#define OP(X)		((X) << 30)
  58#define OP2(X)		((X) << 22)
  59#define OP3(X)		((X) << 19)
  60#define COND(X)		(((X) & 0xf) << 25)
  61#define CBCOND(X)	(((X) & 0x1f) << 25)
  62#define F1(X)		OP(X)
  63#define F2(X, Y)	(OP(X) | OP2(Y))
  64#define F3(X, Y)	(OP(X) | OP3(Y))
  65#define ASI(X)		(((X) & 0xff) << 5)
  66
  67#define CONDN		COND(0x0)
  68#define CONDE		COND(0x1)
  69#define CONDLE		COND(0x2)
  70#define CONDL		COND(0x3)
  71#define CONDLEU		COND(0x4)
  72#define CONDCS		COND(0x5)
  73#define CONDNEG		COND(0x6)
  74#define CONDVC		COND(0x7)
  75#define CONDA		COND(0x8)
  76#define CONDNE		COND(0x9)
  77#define CONDG		COND(0xa)
  78#define CONDGE		COND(0xb)
  79#define CONDGU		COND(0xc)
  80#define CONDCC		COND(0xd)
  81#define CONDPOS		COND(0xe)
  82#define CONDVS		COND(0xf)
  83
  84#define CONDGEU		CONDCC
  85#define CONDLU		CONDCS
  86
  87#define WDISP22(X)	(((X) >> 2) & 0x3fffff)
  88#define WDISP19(X)	(((X) >> 2) & 0x7ffff)
  89
  90/* The 10-bit branch displacement for CBCOND is split into two fields */
  91static u32 WDISP10(u32 off)
  92{
  93	u32 ret = ((off >> 2) & 0xff) << 5;
  94
  95	ret |= ((off >> (2 + 8)) & 0x03) << 19;
  96
  97	return ret;
  98}
  99
 100#define CBCONDE		CBCOND(0x09)
 101#define CBCONDLE	CBCOND(0x0a)
 102#define CBCONDL		CBCOND(0x0b)
 103#define CBCONDLEU	CBCOND(0x0c)
 104#define CBCONDCS	CBCOND(0x0d)
 105#define CBCONDN		CBCOND(0x0e)
 106#define CBCONDVS	CBCOND(0x0f)
 107#define CBCONDNE	CBCOND(0x19)
 108#define CBCONDG		CBCOND(0x1a)
 109#define CBCONDGE	CBCOND(0x1b)
 110#define CBCONDGU	CBCOND(0x1c)
 111#define CBCONDCC	CBCOND(0x1d)
 112#define CBCONDPOS	CBCOND(0x1e)
 113#define CBCONDVC	CBCOND(0x1f)
 114
 115#define CBCONDGEU	CBCONDCC
 116#define CBCONDLU	CBCONDCS
 117
 118#define ANNUL		(1 << 29)
 119#define XCC		(1 << 21)
 120
 121#define BRANCH		(F2(0, 1) | XCC)
 122#define CBCOND_OP	(F2(0, 3) | XCC)
 123
 124#define BA		(BRANCH | CONDA)
 125#define BG		(BRANCH | CONDG)
 126#define BL		(BRANCH | CONDL)
 127#define BLE		(BRANCH | CONDLE)
 128#define BGU		(BRANCH | CONDGU)
 129#define BLEU		(BRANCH | CONDLEU)
 130#define BGE		(BRANCH | CONDGE)
 131#define BGEU		(BRANCH | CONDGEU)
 132#define BLU		(BRANCH | CONDLU)
 133#define BE		(BRANCH | CONDE)
 134#define BNE		(BRANCH | CONDNE)
 135
 136#define SETHI(K, REG)	\
 137	(F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
 138#define OR_LO(K, REG)	\
 139	(F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
 140
 141#define ADD		F3(2, 0x00)
 142#define AND		F3(2, 0x01)
 143#define ANDCC		F3(2, 0x11)
 144#define OR		F3(2, 0x02)
 145#define XOR		F3(2, 0x03)
 146#define SUB		F3(2, 0x04)
 147#define SUBCC		F3(2, 0x14)
 148#define MUL		F3(2, 0x0a)
 149#define MULX		F3(2, 0x09)
 150#define UDIVX		F3(2, 0x0d)
 151#define DIV		F3(2, 0x0e)
 152#define SLL		F3(2, 0x25)
 153#define SLLX		(F3(2, 0x25)|(1<<12))
 154#define SRA		F3(2, 0x27)
 155#define SRAX		(F3(2, 0x27)|(1<<12))
 156#define SRL		F3(2, 0x26)
 157#define SRLX		(F3(2, 0x26)|(1<<12))
 158#define JMPL		F3(2, 0x38)
 159#define SAVE		F3(2, 0x3c)
 160#define RESTORE		F3(2, 0x3d)
 161#define CALL		F1(1)
 162#define BR		F2(0, 0x01)
 163#define RD_Y		F3(2, 0x28)
 164#define WR_Y		F3(2, 0x30)
 165
 166#define LD32		F3(3, 0x00)
 167#define LD8		F3(3, 0x01)
 168#define LD16		F3(3, 0x02)
 169#define LD64		F3(3, 0x0b)
 170#define LD64A		F3(3, 0x1b)
 171#define ST8		F3(3, 0x05)
 172#define ST16		F3(3, 0x06)
 173#define ST32		F3(3, 0x04)
 174#define ST64		F3(3, 0x0e)
 175
 176#define CAS		F3(3, 0x3c)
 177#define CASX		F3(3, 0x3e)
 178
 179#define LDPTR		LD64
 180#define BASE_STACKFRAME	176
 181
 182#define LD32I		(LD32 | IMMED)
 183#define LD8I		(LD8 | IMMED)
 184#define LD16I		(LD16 | IMMED)
 185#define LD64I		(LD64 | IMMED)
 186#define LDPTRI		(LDPTR | IMMED)
 187#define ST32I		(ST32 | IMMED)
 188
 189struct jit_ctx {
 190	struct bpf_prog		*prog;
 191	unsigned int		*offset;
 192	int			idx;
 193	int			epilogue_offset;
 194	bool 			tmp_1_used;
 195	bool 			tmp_2_used;
 196	bool 			tmp_3_used;
 197	bool			saw_frame_pointer;
 198	bool			saw_call;
 199	bool			saw_tail_call;
 200	u32			*image;
 201};
 202
 203#define TMP_REG_1	(MAX_BPF_JIT_REG + 0)
 204#define TMP_REG_2	(MAX_BPF_JIT_REG + 1)
 205#define TMP_REG_3	(MAX_BPF_JIT_REG + 2)
 206
 207/* Map BPF registers to SPARC registers */
 208static const int bpf2sparc[] = {
 209	/* return value from in-kernel function, and exit value from eBPF */
 210	[BPF_REG_0] = O5,
 211
 212	/* arguments from eBPF program to in-kernel function */
 213	[BPF_REG_1] = O0,
 214	[BPF_REG_2] = O1,
 215	[BPF_REG_3] = O2,
 216	[BPF_REG_4] = O3,
 217	[BPF_REG_5] = O4,
 218
 219	/* callee saved registers that in-kernel function will preserve */
 220	[BPF_REG_6] = L0,
 221	[BPF_REG_7] = L1,
 222	[BPF_REG_8] = L2,
 223	[BPF_REG_9] = L3,
 224
 225	/* read-only frame pointer to access stack */
 226	[BPF_REG_FP] = L6,
 227
 228	[BPF_REG_AX] = G7,
 229
 230	/* temporary register for BPF JIT */
 231	[TMP_REG_1] = G1,
 232	[TMP_REG_2] = G2,
 233	[TMP_REG_3] = G3,
 234};
 235
 236static void emit(const u32 insn, struct jit_ctx *ctx)
 237{
 238	if (ctx->image != NULL)
 239		ctx->image[ctx->idx] = insn;
 240
 241	ctx->idx++;
 242}
 243
 244static void emit_call(u32 *func, struct jit_ctx *ctx)
 245{
 246	if (ctx->image != NULL) {
 247		void *here = &ctx->image[ctx->idx];
 248		unsigned int off;
 249
 250		off = (void *)func - here;
 251		ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
 252	}
 253	ctx->idx++;
 254}
 255
 256static void emit_nop(struct jit_ctx *ctx)
 257{
 258	emit(SETHI(0, G0), ctx);
 259}
 260
 261static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
 262{
 263	emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
 264}
 265
 266/* Emit 32-bit constant, zero extended. */
 267static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
 268{
 269	emit(SETHI(K, reg), ctx);
 270	emit(OR_LO(K, reg), ctx);
 271}
 272
 273/* Emit 32-bit constant, sign extended. */
 274static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
 275{
 276	if (K >= 0) {
 277		emit(SETHI(K, reg), ctx);
 278		emit(OR_LO(K, reg), ctx);
 279	} else {
 280		u32 hbits = ~(u32) K;
 281		u32 lbits = -0x400 | (u32) K;
 282
 283		emit(SETHI(hbits, reg), ctx);
 284		emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
 285	}
 286}
 287
 288static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
 289{
 290	emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
 291}
 292
 293static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
 294{
 295	emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
 296}
 297
 298static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
 299		       struct jit_ctx *ctx)
 300{
 301	bool small_immed = is_simm13(imm);
 302	unsigned int insn = opcode;
 303
 304	insn |= RS1(dst) | RD(dst);
 305	if (small_immed) {
 306		emit(insn | IMMED | S13(imm), ctx);
 307	} else {
 308		unsigned int tmp = bpf2sparc[TMP_REG_1];
 309
 310		ctx->tmp_1_used = true;
 311
 312		emit_set_const_sext(imm, tmp, ctx);
 313		emit(insn | RS2(tmp), ctx);
 314	}
 315}
 316
 317static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
 318			unsigned int dst, struct jit_ctx *ctx)
 319{
 320	bool small_immed = is_simm13(imm);
 321	unsigned int insn = opcode;
 322
 323	insn |= RS1(src) | RD(dst);
 324	if (small_immed) {
 325		emit(insn | IMMED | S13(imm), ctx);
 326	} else {
 327		unsigned int tmp = bpf2sparc[TMP_REG_1];
 328
 329		ctx->tmp_1_used = true;
 330
 331		emit_set_const_sext(imm, tmp, ctx);
 332		emit(insn | RS2(tmp), ctx);
 333	}
 334}
 335
 336static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
 337{
 338	if (K >= 0 && is_simm13(K)) {
 339		/* or %g0, K, DEST */
 340		emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 341	} else {
 342		emit_set_const(K, dest, ctx);
 343	}
 344}
 345
 346static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
 347{
 348	if (is_simm13(K)) {
 349		/* or %g0, K, DEST */
 350		emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 351	} else {
 352		emit_set_const(K, dest, ctx);
 353	}
 354}
 355
 356static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
 357{
 358	if (is_simm13(K)) {
 359		/* or %g0, K, DEST */
 360		emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 361	} else {
 362		emit_set_const_sext(K, dest, ctx);
 363	}
 364}
 365
 366static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
 367				   int *hbsp, int *lbsp, int *abbasp)
 368{
 369	int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
 370	int i;
 371
 372	lowest_bit_set = highest_bit_set = -1;
 373	i = 0;
 374	do {
 375		if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
 376			lowest_bit_set = i;
 377		if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
 378			highest_bit_set = (64 - i - 1);
 379	}  while (++i < 32 && (highest_bit_set == -1 ||
 380			       lowest_bit_set == -1));
 381	if (i == 32) {
 382		i = 0;
 383		do {
 384			if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
 385				lowest_bit_set = i + 32;
 386			if (highest_bit_set == -1 &&
 387			    ((low_bits >> (32 - i - 1)) & 1))
 388				highest_bit_set = 32 - i - 1;
 389		} while (++i < 32 && (highest_bit_set == -1 ||
 390				      lowest_bit_set == -1));
 391	}
 392
 393	all_bits_between_are_set = 1;
 394	for (i = lowest_bit_set; i <= highest_bit_set; i++) {
 395		if (i < 32) {
 396			if ((low_bits & (1 << i)) != 0)
 397				continue;
 398		} else {
 399			if ((high_bits & (1 << (i - 32))) != 0)
 400				continue;
 401		}
 402		all_bits_between_are_set = 0;
 403		break;
 404	}
 405	*hbsp = highest_bit_set;
 406	*lbsp = lowest_bit_set;
 407	*abbasp = all_bits_between_are_set;
 408}
 409
 410static unsigned long create_simple_focus_bits(unsigned long high_bits,
 411					      unsigned long low_bits,
 412					      int lowest_bit_set, int shift)
 413{
 414	long hi, lo;
 415
 416	if (lowest_bit_set < 32) {
 417		lo = (low_bits >> lowest_bit_set) << shift;
 418		hi = ((high_bits << (32 - lowest_bit_set)) << shift);
 419	} else {
 420		lo = 0;
 421		hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
 422	}
 423	return hi | lo;
 424}
 425
 426static bool const64_is_2insns(unsigned long high_bits,
 427			      unsigned long low_bits)
 428{
 429	int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
 430
 431	if (high_bits == 0 || high_bits == 0xffffffff)
 432		return true;
 433
 434	analyze_64bit_constant(high_bits, low_bits,
 435			       &highest_bit_set, &lowest_bit_set,
 436			       &all_bits_between_are_set);
 437
 438	if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
 439	    all_bits_between_are_set != 0)
 440		return true;
 441
 442	if (highest_bit_set - lowest_bit_set < 21)
 443		return true;
 444
 445	return false;
 446}
 447
 448static void sparc_emit_set_const64_quick2(unsigned long high_bits,
 449					  unsigned long low_imm,
 450					  unsigned int dest,
 451					  int shift_count, struct jit_ctx *ctx)
 452{
 453	emit_loadimm32(high_bits, dest, ctx);
 454
 455	/* Now shift it up into place.  */
 456	emit_alu_K(SLLX, dest, shift_count, ctx);
 457
 458	/* If there is a low immediate part piece, finish up by
 459	 * putting that in as well.
 460	 */
 461	if (low_imm != 0)
 462		emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
 463}
 464
 465static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
 466{
 467	int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
 468	unsigned int tmp = bpf2sparc[TMP_REG_1];
 469	u32 low_bits = (K & 0xffffffff);
 470	u32 high_bits = (K >> 32);
 471
 472	/* These two tests also take care of all of the one
 473	 * instruction cases.
 474	 */
 475	if (high_bits == 0xffffffff && (low_bits & 0x80000000))
 476		return emit_loadimm_sext(K, dest, ctx);
 477	if (high_bits == 0x00000000)
 478		return emit_loadimm32(K, dest, ctx);
 479
 480	analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
 481			       &lowest_bit_set, &all_bits_between_are_set);
 482
 483	/* 1) mov	-1, %reg
 484	 *    sllx	%reg, shift, %reg
 485	 * 2) mov	-1, %reg
 486	 *    srlx	%reg, shift, %reg
 487	 * 3) mov	some_small_const, %reg
 488	 *    sllx	%reg, shift, %reg
 489	 */
 490	if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
 491	     all_bits_between_are_set != 0) ||
 492	    ((highest_bit_set - lowest_bit_set) < 12)) {
 493		int shift = lowest_bit_set;
 494		long the_const = -1;
 495
 496		if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
 497		    all_bits_between_are_set == 0) {
 498			the_const =
 499				create_simple_focus_bits(high_bits, low_bits,
 500							 lowest_bit_set, 0);
 501		} else if (lowest_bit_set == 0)
 502			shift = -(63 - highest_bit_set);
 503
 504		emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
 505		if (shift > 0)
 506			emit_alu_K(SLLX, dest, shift, ctx);
 507		else if (shift < 0)
 508			emit_alu_K(SRLX, dest, -shift, ctx);
 509
 510		return;
 511	}
 512
 513	/* Now a range of 22 or less bits set somewhere.
 514	 * 1) sethi	%hi(focus_bits), %reg
 515	 *    sllx	%reg, shift, %reg
 516	 * 2) sethi	%hi(focus_bits), %reg
 517	 *    srlx	%reg, shift, %reg
 518	 */
 519	if ((highest_bit_set - lowest_bit_set) < 21) {
 520		unsigned long focus_bits =
 521			create_simple_focus_bits(high_bits, low_bits,
 522						 lowest_bit_set, 10);
 523
 524		emit(SETHI(focus_bits, dest), ctx);
 525
 526		/* If lowest_bit_set == 10 then a sethi alone could
 527		 * have done it.
 528		 */
 529		if (lowest_bit_set < 10)
 530			emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
 531		else if (lowest_bit_set > 10)
 532			emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
 533		return;
 534	}
 535
 536	/* Ok, now 3 instruction sequences.  */
 537	if (low_bits == 0) {
 538		emit_loadimm32(high_bits, dest, ctx);
 539		emit_alu_K(SLLX, dest, 32, ctx);
 540		return;
 541	}
 542
 543	/* We may be able to do something quick
 544	 * when the constant is negated, so try that.
 545	 */
 546	if (const64_is_2insns((~high_bits) & 0xffffffff,
 547			      (~low_bits) & 0xfffffc00)) {
 548		/* NOTE: The trailing bits get XOR'd so we need the
 549		 * non-negated bits, not the negated ones.
 550		 */
 551		unsigned long trailing_bits = low_bits & 0x3ff;
 552
 553		if ((((~high_bits) & 0xffffffff) == 0 &&
 554		     ((~low_bits) & 0x80000000) == 0) ||
 555		    (((~high_bits) & 0xffffffff) == 0xffffffff &&
 556		     ((~low_bits) & 0x80000000) != 0)) {
 557			unsigned long fast_int = (~low_bits & 0xffffffff);
 558
 559			if ((is_sethi(fast_int) &&
 560			     (~high_bits & 0xffffffff) == 0)) {
 561				emit(SETHI(fast_int, dest), ctx);
 562			} else if (is_simm13(fast_int)) {
 563				emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
 564			} else {
 565				emit_loadimm64(fast_int, dest, ctx);
 566			}
 567		} else {
 568			u64 n = ((~low_bits) & 0xfffffc00) |
 569				(((unsigned long)((~high_bits) & 0xffffffff))<<32);
 570			emit_loadimm64(n, dest, ctx);
 571		}
 572
 573		low_bits = -0x400 | trailing_bits;
 574
 575		emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
 576		return;
 577	}
 578
 579	/* 1) sethi	%hi(xxx), %reg
 580	 *    or	%reg, %lo(xxx), %reg
 581	 *    sllx	%reg, yyy, %reg
 582	 */
 583	if ((highest_bit_set - lowest_bit_set) < 32) {
 584		unsigned long focus_bits =
 585			create_simple_focus_bits(high_bits, low_bits,
 586						 lowest_bit_set, 0);
 587
 588		/* So what we know is that the set bits straddle the
 589		 * middle of the 64-bit word.
 590		 */
 591		sparc_emit_set_const64_quick2(focus_bits, 0, dest,
 592					      lowest_bit_set, ctx);
 593		return;
 594	}
 595
 596	/* 1) sethi	%hi(high_bits), %reg
 597	 *    or	%reg, %lo(high_bits), %reg
 598	 *    sllx	%reg, 32, %reg
 599	 *    or	%reg, low_bits, %reg
 600	 */
 601	if (is_simm13(low_bits) && ((int)low_bits > 0)) {
 602		sparc_emit_set_const64_quick2(high_bits, low_bits,
 603					      dest, 32, ctx);
 604		return;
 605	}
 606
 607	/* Oh well, we tried... Do a full 64-bit decomposition.  */
 608	ctx->tmp_1_used = true;
 609
 610	emit_loadimm32(high_bits, tmp, ctx);
 611	emit_loadimm32(low_bits, dest, ctx);
 612	emit_alu_K(SLLX, tmp, 32, ctx);
 613	emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
 614}
 615
 616static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
 617			struct jit_ctx *ctx)
 618{
 619	unsigned int off = to_idx - from_idx;
 620
 621	if (br_opc & XCC)
 622		emit(br_opc | WDISP19(off << 2), ctx);
 623	else
 624		emit(br_opc | WDISP22(off << 2), ctx);
 625}
 626
 627static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 628			const u8 dst, const u8 src, struct jit_ctx *ctx)
 629{
 630	unsigned int off = to_idx - from_idx;
 631
 632	emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
 633}
 634
 635static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 636			 const u8 dst, s32 imm, struct jit_ctx *ctx)
 637{
 638	unsigned int off = to_idx - from_idx;
 639
 640	emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
 641}
 642
 643#define emit_read_y(REG, CTX)	emit(RD_Y | RD(REG), CTX)
 644#define emit_write_y(REG, CTX)	emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
 645
 646#define emit_cmp(R1, R2, CTX)				\
 647	emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 648
 649#define emit_cmpi(R1, IMM, CTX)				\
 650	emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 651
 652#define emit_btst(R1, R2, CTX)				\
 653	emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 654
 655#define emit_btsti(R1, IMM, CTX)			\
 656	emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 657
 658static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
 659				   const s32 imm, bool is_imm, int branch_dst,
 660				   struct jit_ctx *ctx)
 661{
 662	bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
 663	const u8 tmp = bpf2sparc[TMP_REG_1];
 664
 665	branch_dst = ctx->offset[branch_dst];
 666
 667	if (!is_simm10(branch_dst - ctx->idx) ||
 668	    BPF_OP(code) == BPF_JSET)
 669		use_cbcond = false;
 670
 671	if (is_imm) {
 672		bool fits = true;
 673
 674		if (use_cbcond) {
 675			if (!is_simm5(imm))
 676				fits = false;
 677		} else if (!is_simm13(imm)) {
 678			fits = false;
 679		}
 680		if (!fits) {
 681			ctx->tmp_1_used = true;
 682			emit_loadimm_sext(imm, tmp, ctx);
 683			src = tmp;
 684			is_imm = false;
 685		}
 686	}
 687
 688	if (!use_cbcond) {
 689		u32 br_opcode;
 690
 691		if (BPF_OP(code) == BPF_JSET) {
 692			if (is_imm)
 693				emit_btsti(dst, imm, ctx);
 694			else
 695				emit_btst(dst, src, ctx);
 696		} else {
 697			if (is_imm)
 698				emit_cmpi(dst, imm, ctx);
 699			else
 700				emit_cmp(dst, src, ctx);
 701		}
 702		switch (BPF_OP(code)) {
 703		case BPF_JEQ:
 704			br_opcode = BE;
 705			break;
 706		case BPF_JGT:
 707			br_opcode = BGU;
 708			break;
 709		case BPF_JLT:
 710			br_opcode = BLU;
 711			break;
 712		case BPF_JGE:
 713			br_opcode = BGEU;
 714			break;
 715		case BPF_JLE:
 716			br_opcode = BLEU;
 717			break;
 718		case BPF_JSET:
 719		case BPF_JNE:
 720			br_opcode = BNE;
 721			break;
 722		case BPF_JSGT:
 723			br_opcode = BG;
 724			break;
 725		case BPF_JSLT:
 726			br_opcode = BL;
 727			break;
 728		case BPF_JSGE:
 729			br_opcode = BGE;
 730			break;
 731		case BPF_JSLE:
 732			br_opcode = BLE;
 733			break;
 734		default:
 735			/* Make sure we dont leak kernel information to the
 736			 * user.
 737			 */
 738			return -EFAULT;
 739		}
 740		emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
 741		emit_nop(ctx);
 742	} else {
 743		u32 cbcond_opcode;
 744
 745		switch (BPF_OP(code)) {
 746		case BPF_JEQ:
 747			cbcond_opcode = CBCONDE;
 748			break;
 749		case BPF_JGT:
 750			cbcond_opcode = CBCONDGU;
 751			break;
 752		case BPF_JLT:
 753			cbcond_opcode = CBCONDLU;
 754			break;
 755		case BPF_JGE:
 756			cbcond_opcode = CBCONDGEU;
 757			break;
 758		case BPF_JLE:
 759			cbcond_opcode = CBCONDLEU;
 760			break;
 761		case BPF_JNE:
 762			cbcond_opcode = CBCONDNE;
 763			break;
 764		case BPF_JSGT:
 765			cbcond_opcode = CBCONDG;
 766			break;
 767		case BPF_JSLT:
 768			cbcond_opcode = CBCONDL;
 769			break;
 770		case BPF_JSGE:
 771			cbcond_opcode = CBCONDGE;
 772			break;
 773		case BPF_JSLE:
 774			cbcond_opcode = CBCONDLE;
 775			break;
 776		default:
 777			/* Make sure we dont leak kernel information to the
 778			 * user.
 779			 */
 780			return -EFAULT;
 781		}
 782		cbcond_opcode |= CBCOND_OP;
 783		if (is_imm)
 784			emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
 785				     dst, imm, ctx);
 786		else
 787			emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
 788				    dst, src, ctx);
 789	}
 790	return 0;
 791}
 792
 793/* Just skip the save instruction and the ctx register move.  */
 794#define BPF_TAILCALL_PROLOGUE_SKIP	32
 795#define BPF_TAILCALL_CNT_SP_OFF		(STACK_BIAS + 128)
 796
 797static void build_prologue(struct jit_ctx *ctx)
 798{
 799	s32 stack_needed = BASE_STACKFRAME;
 800
 801	if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
 802		struct bpf_prog *prog = ctx->prog;
 803		u32 stack_depth;
 804
 805		stack_depth = prog->aux->stack_depth;
 806		stack_needed += round_up(stack_depth, 16);
 807	}
 808
 809	if (ctx->saw_tail_call)
 810		stack_needed += 8;
 811
 812	/* save %sp, -176, %sp */
 813	emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
 814
 815	/* tail_call_cnt = 0 */
 816	if (ctx->saw_tail_call) {
 817		u32 off = BPF_TAILCALL_CNT_SP_OFF;
 818
 819		emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
 820	} else {
 821		emit_nop(ctx);
 822	}
 823	if (ctx->saw_frame_pointer) {
 824		const u8 vfp = bpf2sparc[BPF_REG_FP];
 825
 826		emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
 827	} else {
 828		emit_nop(ctx);
 829	}
 830
 831	emit_reg_move(I0, O0, ctx);
 832	emit_reg_move(I1, O1, ctx);
 833	emit_reg_move(I2, O2, ctx);
 834	emit_reg_move(I3, O3, ctx);
 835	emit_reg_move(I4, O4, ctx);
 836	/* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
 837}
 838
 839static void build_epilogue(struct jit_ctx *ctx)
 840{
 841	ctx->epilogue_offset = ctx->idx;
 842
 843	/* ret (jmpl %i7 + 8, %g0) */
 844	emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
 845
 846	/* restore %i5, %g0, %o0 */
 847	emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
 848}
 849
 850static void emit_tail_call(struct jit_ctx *ctx)
 851{
 852	const u8 bpf_array = bpf2sparc[BPF_REG_2];
 853	const u8 bpf_index = bpf2sparc[BPF_REG_3];
 854	const u8 tmp = bpf2sparc[TMP_REG_1];
 855	u32 off;
 856
 857	ctx->saw_tail_call = true;
 858
 859	off = offsetof(struct bpf_array, map.max_entries);
 860	emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
 861	emit_cmp(bpf_index, tmp, ctx);
 862#define OFFSET1 17
 863	emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
 864	emit_nop(ctx);
 865
 866	off = BPF_TAILCALL_CNT_SP_OFF;
 867	emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 868	emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
 869#define OFFSET2 13
 870	emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx);
 871	emit_nop(ctx);
 872
 873	emit_alu_K(ADD, tmp, 1, ctx);
 874	off = BPF_TAILCALL_CNT_SP_OFF;
 875	emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 876
 877	emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
 878	emit_alu(ADD, bpf_array, tmp, ctx);
 879	off = offsetof(struct bpf_array, ptrs);
 880	emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 881
 882	emit_cmpi(tmp, 0, ctx);
 883#define OFFSET3 5
 884	emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
 885	emit_nop(ctx);
 886
 887	off = offsetof(struct bpf_prog, bpf_func);
 888	emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 889
 890	off = BPF_TAILCALL_PROLOGUE_SKIP;
 891	emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
 892	emit_nop(ctx);
 893}
 894
 895static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 896{
 897	const u8 code = insn->code;
 898	const u8 dst = bpf2sparc[insn->dst_reg];
 899	const u8 src = bpf2sparc[insn->src_reg];
 900	const int i = insn - ctx->prog->insnsi;
 901	const s16 off = insn->off;
 902	const s32 imm = insn->imm;
 903
 904	if (insn->src_reg == BPF_REG_FP)
 905		ctx->saw_frame_pointer = true;
 906
 907	switch (code) {
 908	/* dst = src */
 909	case BPF_ALU | BPF_MOV | BPF_X:
 910		emit_alu3_K(SRL, src, 0, dst, ctx);
 911		if (insn_is_zext(&insn[1]))
 912			return 1;
 913		break;
 914	case BPF_ALU64 | BPF_MOV | BPF_X:
 915		emit_reg_move(src, dst, ctx);
 916		break;
 917	/* dst = dst OP src */
 918	case BPF_ALU | BPF_ADD | BPF_X:
 919	case BPF_ALU64 | BPF_ADD | BPF_X:
 920		emit_alu(ADD, src, dst, ctx);
 921		goto do_alu32_trunc;
 922	case BPF_ALU | BPF_SUB | BPF_X:
 923	case BPF_ALU64 | BPF_SUB | BPF_X:
 924		emit_alu(SUB, src, dst, ctx);
 925		goto do_alu32_trunc;
 926	case BPF_ALU | BPF_AND | BPF_X:
 927	case BPF_ALU64 | BPF_AND | BPF_X:
 928		emit_alu(AND, src, dst, ctx);
 929		goto do_alu32_trunc;
 930	case BPF_ALU | BPF_OR | BPF_X:
 931	case BPF_ALU64 | BPF_OR | BPF_X:
 932		emit_alu(OR, src, dst, ctx);
 933		goto do_alu32_trunc;
 934	case BPF_ALU | BPF_XOR | BPF_X:
 935	case BPF_ALU64 | BPF_XOR | BPF_X:
 936		emit_alu(XOR, src, dst, ctx);
 937		goto do_alu32_trunc;
 938	case BPF_ALU | BPF_MUL | BPF_X:
 939		emit_alu(MUL, src, dst, ctx);
 940		goto do_alu32_trunc;
 941	case BPF_ALU64 | BPF_MUL | BPF_X:
 942		emit_alu(MULX, src, dst, ctx);
 943		break;
 944	case BPF_ALU | BPF_DIV | BPF_X:
 945		emit_write_y(G0, ctx);
 946		emit_alu(DIV, src, dst, ctx);
 947		if (insn_is_zext(&insn[1]))
 948			return 1;
 949		break;
 950	case BPF_ALU64 | BPF_DIV | BPF_X:
 951		emit_alu(UDIVX, src, dst, ctx);
 952		break;
 953	case BPF_ALU | BPF_MOD | BPF_X: {
 954		const u8 tmp = bpf2sparc[TMP_REG_1];
 955
 956		ctx->tmp_1_used = true;
 957
 958		emit_write_y(G0, ctx);
 959		emit_alu3(DIV, dst, src, tmp, ctx);
 960		emit_alu3(MULX, tmp, src, tmp, ctx);
 961		emit_alu3(SUB, dst, tmp, dst, ctx);
 962		goto do_alu32_trunc;
 963	}
 964	case BPF_ALU64 | BPF_MOD | BPF_X: {
 965		const u8 tmp = bpf2sparc[TMP_REG_1];
 966
 967		ctx->tmp_1_used = true;
 968
 969		emit_alu3(UDIVX, dst, src, tmp, ctx);
 970		emit_alu3(MULX, tmp, src, tmp, ctx);
 971		emit_alu3(SUB, dst, tmp, dst, ctx);
 972		break;
 973	}
 974	case BPF_ALU | BPF_LSH | BPF_X:
 975		emit_alu(SLL, src, dst, ctx);
 976		goto do_alu32_trunc;
 977	case BPF_ALU64 | BPF_LSH | BPF_X:
 978		emit_alu(SLLX, src, dst, ctx);
 979		break;
 980	case BPF_ALU | BPF_RSH | BPF_X:
 981		emit_alu(SRL, src, dst, ctx);
 982		if (insn_is_zext(&insn[1]))
 983			return 1;
 984		break;
 985	case BPF_ALU64 | BPF_RSH | BPF_X:
 986		emit_alu(SRLX, src, dst, ctx);
 987		break;
 988	case BPF_ALU | BPF_ARSH | BPF_X:
 989		emit_alu(SRA, src, dst, ctx);
 990		goto do_alu32_trunc;
 991	case BPF_ALU64 | BPF_ARSH | BPF_X:
 992		emit_alu(SRAX, src, dst, ctx);
 993		break;
 994
 995	/* dst = -dst */
 996	case BPF_ALU | BPF_NEG:
 997	case BPF_ALU64 | BPF_NEG:
 998		emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
 999		goto do_alu32_trunc;
1000
1001	case BPF_ALU | BPF_END | BPF_FROM_BE:
1002		switch (imm) {
1003		case 16:
1004			emit_alu_K(SLL, dst, 16, ctx);
1005			emit_alu_K(SRL, dst, 16, ctx);
1006			if (insn_is_zext(&insn[1]))
1007				return 1;
1008			break;
1009		case 32:
1010			if (!ctx->prog->aux->verifier_zext)
1011				emit_alu_K(SRL, dst, 0, ctx);
1012			break;
1013		case 64:
1014			/* nop */
1015			break;
1016
1017		}
1018		break;
1019
1020	/* dst = BSWAP##imm(dst) */
1021	case BPF_ALU | BPF_END | BPF_FROM_LE: {
1022		const u8 tmp = bpf2sparc[TMP_REG_1];
1023		const u8 tmp2 = bpf2sparc[TMP_REG_2];
1024
1025		ctx->tmp_1_used = true;
1026		switch (imm) {
1027		case 16:
1028			emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1029			emit_alu3_K(SRL, dst, 8, dst, ctx);
1030			emit_alu3_K(AND, dst, 0xff, dst, ctx);
1031			emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1032			emit_alu(OR, tmp, dst, ctx);
1033			if (insn_is_zext(&insn[1]))
1034				return 1;
1035			break;
1036
1037		case 32:
1038			ctx->tmp_2_used = true;
1039			emit_alu3_K(SRL, dst, 24, tmp, ctx);	/* tmp  = dst >> 24 */
1040			emit_alu3_K(SRL, dst, 16, tmp2, ctx);	/* tmp2 = dst >> 16 */
1041			emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1042			emit_alu3_K(SLL, tmp2, 8, tmp2, ctx);	/* tmp2 = tmp2 << 8 */
1043			emit_alu(OR, tmp2, tmp, ctx);		/* tmp  = tmp | tmp2 */
1044			emit_alu3_K(SRL, dst, 8, tmp2, ctx);	/* tmp2 = dst >> 8 */
1045			emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1046			emit_alu3_K(SLL, tmp2, 16, tmp2, ctx);	/* tmp2 = tmp2 << 16 */
1047			emit_alu(OR, tmp2, tmp, ctx);		/* tmp  = tmp | tmp2 */
1048			emit_alu3_K(AND, dst, 0xff, dst, ctx);	/* dst	= dst & 0xff */
1049			emit_alu3_K(SLL, dst, 24, dst, ctx);	/* dst  = dst << 24 */
1050			emit_alu(OR, tmp, dst, ctx);		/* dst  = dst | tmp */
1051			if (insn_is_zext(&insn[1]))
1052				return 1;
1053			break;
1054
1055		case 64:
1056			emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1057			emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1058			emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1059			break;
1060		}
1061		break;
1062	}
1063	/* dst = imm */
1064	case BPF_ALU | BPF_MOV | BPF_K:
1065		emit_loadimm32(imm, dst, ctx);
1066		if (insn_is_zext(&insn[1]))
1067			return 1;
1068		break;
1069	case BPF_ALU64 | BPF_MOV | BPF_K:
1070		emit_loadimm_sext(imm, dst, ctx);
1071		break;
1072	/* dst = dst OP imm */
1073	case BPF_ALU | BPF_ADD | BPF_K:
1074	case BPF_ALU64 | BPF_ADD | BPF_K:
1075		emit_alu_K(ADD, dst, imm, ctx);
1076		goto do_alu32_trunc;
1077	case BPF_ALU | BPF_SUB | BPF_K:
1078	case BPF_ALU64 | BPF_SUB | BPF_K:
1079		emit_alu_K(SUB, dst, imm, ctx);
1080		goto do_alu32_trunc;
1081	case BPF_ALU | BPF_AND | BPF_K:
1082	case BPF_ALU64 | BPF_AND | BPF_K:
1083		emit_alu_K(AND, dst, imm, ctx);
1084		goto do_alu32_trunc;
1085	case BPF_ALU | BPF_OR | BPF_K:
1086	case BPF_ALU64 | BPF_OR | BPF_K:
1087		emit_alu_K(OR, dst, imm, ctx);
1088		goto do_alu32_trunc;
1089	case BPF_ALU | BPF_XOR | BPF_K:
1090	case BPF_ALU64 | BPF_XOR | BPF_K:
1091		emit_alu_K(XOR, dst, imm, ctx);
1092		goto do_alu32_trunc;
1093	case BPF_ALU | BPF_MUL | BPF_K:
1094		emit_alu_K(MUL, dst, imm, ctx);
1095		goto do_alu32_trunc;
1096	case BPF_ALU64 | BPF_MUL | BPF_K:
1097		emit_alu_K(MULX, dst, imm, ctx);
1098		break;
1099	case BPF_ALU | BPF_DIV | BPF_K:
1100		if (imm == 0)
1101			return -EINVAL;
1102
1103		emit_write_y(G0, ctx);
1104		emit_alu_K(DIV, dst, imm, ctx);
1105		goto do_alu32_trunc;
1106	case BPF_ALU64 | BPF_DIV | BPF_K:
1107		if (imm == 0)
1108			return -EINVAL;
1109
1110		emit_alu_K(UDIVX, dst, imm, ctx);
1111		break;
1112	case BPF_ALU64 | BPF_MOD | BPF_K:
1113	case BPF_ALU | BPF_MOD | BPF_K: {
1114		const u8 tmp = bpf2sparc[TMP_REG_2];
1115		unsigned int div;
1116
1117		if (imm == 0)
1118			return -EINVAL;
1119
1120		div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1121
1122		ctx->tmp_2_used = true;
1123
1124		if (BPF_CLASS(code) != BPF_ALU64)
1125			emit_write_y(G0, ctx);
1126		if (is_simm13(imm)) {
1127			emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1128			emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1129			emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1130		} else {
1131			const u8 tmp1 = bpf2sparc[TMP_REG_1];
1132
1133			ctx->tmp_1_used = true;
1134
1135			emit_set_const_sext(imm, tmp1, ctx);
1136			emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1137			emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1138			emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1139		}
1140		goto do_alu32_trunc;
1141	}
1142	case BPF_ALU | BPF_LSH | BPF_K:
1143		emit_alu_K(SLL, dst, imm, ctx);
1144		goto do_alu32_trunc;
1145	case BPF_ALU64 | BPF_LSH | BPF_K:
1146		emit_alu_K(SLLX, dst, imm, ctx);
1147		break;
1148	case BPF_ALU | BPF_RSH | BPF_K:
1149		emit_alu_K(SRL, dst, imm, ctx);
1150		if (insn_is_zext(&insn[1]))
1151			return 1;
1152		break;
1153	case BPF_ALU64 | BPF_RSH | BPF_K:
1154		emit_alu_K(SRLX, dst, imm, ctx);
1155		break;
1156	case BPF_ALU | BPF_ARSH | BPF_K:
1157		emit_alu_K(SRA, dst, imm, ctx);
1158		goto do_alu32_trunc;
1159	case BPF_ALU64 | BPF_ARSH | BPF_K:
1160		emit_alu_K(SRAX, dst, imm, ctx);
1161		break;
1162
1163	do_alu32_trunc:
1164		if (BPF_CLASS(code) == BPF_ALU &&
1165		    !ctx->prog->aux->verifier_zext)
1166			emit_alu_K(SRL, dst, 0, ctx);
1167		break;
1168
1169	/* JUMP off */
1170	case BPF_JMP | BPF_JA:
1171		emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1172		emit_nop(ctx);
1173		break;
1174	/* IF (dst COND src) JUMP off */
1175	case BPF_JMP | BPF_JEQ | BPF_X:
1176	case BPF_JMP | BPF_JGT | BPF_X:
1177	case BPF_JMP | BPF_JLT | BPF_X:
1178	case BPF_JMP | BPF_JGE | BPF_X:
1179	case BPF_JMP | BPF_JLE | BPF_X:
1180	case BPF_JMP | BPF_JNE | BPF_X:
1181	case BPF_JMP | BPF_JSGT | BPF_X:
1182	case BPF_JMP | BPF_JSLT | BPF_X:
1183	case BPF_JMP | BPF_JSGE | BPF_X:
1184	case BPF_JMP | BPF_JSLE | BPF_X:
1185	case BPF_JMP | BPF_JSET | BPF_X: {
1186		int err;
1187
1188		err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1189		if (err)
1190			return err;
1191		break;
1192	}
1193	/* IF (dst COND imm) JUMP off */
1194	case BPF_JMP | BPF_JEQ | BPF_K:
1195	case BPF_JMP | BPF_JGT | BPF_K:
1196	case BPF_JMP | BPF_JLT | BPF_K:
1197	case BPF_JMP | BPF_JGE | BPF_K:
1198	case BPF_JMP | BPF_JLE | BPF_K:
1199	case BPF_JMP | BPF_JNE | BPF_K:
1200	case BPF_JMP | BPF_JSGT | BPF_K:
1201	case BPF_JMP | BPF_JSLT | BPF_K:
1202	case BPF_JMP | BPF_JSGE | BPF_K:
1203	case BPF_JMP | BPF_JSLE | BPF_K:
1204	case BPF_JMP | BPF_JSET | BPF_K: {
1205		int err;
1206
1207		err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1208		if (err)
1209			return err;
1210		break;
1211	}
1212
1213	/* function call */
1214	case BPF_JMP | BPF_CALL:
1215	{
1216		u8 *func = ((u8 *)__bpf_call_base) + imm;
1217
1218		ctx->saw_call = true;
1219
1220		emit_call((u32 *)func, ctx);
1221		emit_nop(ctx);
1222
1223		emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1224		break;
1225	}
1226
1227	/* tail call */
1228	case BPF_JMP | BPF_TAIL_CALL:
1229		emit_tail_call(ctx);
1230		break;
1231
1232	/* function return */
1233	case BPF_JMP | BPF_EXIT:
1234		/* Optimization: when last instruction is EXIT,
1235		   simply fallthrough to epilogue. */
1236		if (i == ctx->prog->len - 1)
1237			break;
1238		emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1239		emit_nop(ctx);
1240		break;
1241
1242	/* dst = imm64 */
1243	case BPF_LD | BPF_IMM | BPF_DW:
1244	{
1245		const struct bpf_insn insn1 = insn[1];
1246		u64 imm64;
1247
1248		imm64 = (u64)insn1.imm << 32 | (u32)imm;
1249		emit_loadimm64(imm64, dst, ctx);
1250
1251		return 1;
1252	}
1253
1254	/* LDX: dst = *(size *)(src + off) */
1255	case BPF_LDX | BPF_MEM | BPF_W:
1256	case BPF_LDX | BPF_MEM | BPF_H:
1257	case BPF_LDX | BPF_MEM | BPF_B:
1258	case BPF_LDX | BPF_MEM | BPF_DW: {
1259		const u8 tmp = bpf2sparc[TMP_REG_1];
1260		u32 opcode = 0, rs2;
1261
1262		ctx->tmp_1_used = true;
1263		switch (BPF_SIZE(code)) {
1264		case BPF_W:
1265			opcode = LD32;
1266			break;
1267		case BPF_H:
1268			opcode = LD16;
1269			break;
1270		case BPF_B:
1271			opcode = LD8;
1272			break;
1273		case BPF_DW:
1274			opcode = LD64;
1275			break;
1276		}
1277
1278		if (is_simm13(off)) {
1279			opcode |= IMMED;
1280			rs2 = S13(off);
1281		} else {
1282			emit_loadimm(off, tmp, ctx);
1283			rs2 = RS2(tmp);
1284		}
1285		emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1286		if (opcode != LD64 && insn_is_zext(&insn[1]))
1287			return 1;
1288		break;
1289	}
1290	/* speculation barrier */
1291	case BPF_ST | BPF_NOSPEC:
1292		break;
1293	/* ST: *(size *)(dst + off) = imm */
1294	case BPF_ST | BPF_MEM | BPF_W:
1295	case BPF_ST | BPF_MEM | BPF_H:
1296	case BPF_ST | BPF_MEM | BPF_B:
1297	case BPF_ST | BPF_MEM | BPF_DW: {
1298		const u8 tmp = bpf2sparc[TMP_REG_1];
1299		const u8 tmp2 = bpf2sparc[TMP_REG_2];
1300		u32 opcode = 0, rs2;
1301
1302		if (insn->dst_reg == BPF_REG_FP)
1303			ctx->saw_frame_pointer = true;
1304
1305		ctx->tmp_2_used = true;
1306		emit_loadimm(imm, tmp2, ctx);
1307
1308		switch (BPF_SIZE(code)) {
1309		case BPF_W:
1310			opcode = ST32;
1311			break;
1312		case BPF_H:
1313			opcode = ST16;
1314			break;
1315		case BPF_B:
1316			opcode = ST8;
1317			break;
1318		case BPF_DW:
1319			opcode = ST64;
1320			break;
1321		}
1322
1323		if (is_simm13(off)) {
1324			opcode |= IMMED;
1325			rs2 = S13(off);
1326		} else {
1327			ctx->tmp_1_used = true;
1328			emit_loadimm(off, tmp, ctx);
1329			rs2 = RS2(tmp);
1330		}
1331		emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1332		break;
1333	}
1334
1335	/* STX: *(size *)(dst + off) = src */
1336	case BPF_STX | BPF_MEM | BPF_W:
1337	case BPF_STX | BPF_MEM | BPF_H:
1338	case BPF_STX | BPF_MEM | BPF_B:
1339	case BPF_STX | BPF_MEM | BPF_DW: {
1340		const u8 tmp = bpf2sparc[TMP_REG_1];
1341		u32 opcode = 0, rs2;
1342
1343		if (insn->dst_reg == BPF_REG_FP)
1344			ctx->saw_frame_pointer = true;
1345
1346		switch (BPF_SIZE(code)) {
1347		case BPF_W:
1348			opcode = ST32;
1349			break;
1350		case BPF_H:
1351			opcode = ST16;
1352			break;
1353		case BPF_B:
1354			opcode = ST8;
1355			break;
1356		case BPF_DW:
1357			opcode = ST64;
1358			break;
1359		}
1360		if (is_simm13(off)) {
1361			opcode |= IMMED;
1362			rs2 = S13(off);
1363		} else {
1364			ctx->tmp_1_used = true;
1365			emit_loadimm(off, tmp, ctx);
1366			rs2 = RS2(tmp);
1367		}
1368		emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1369		break;
1370	}
1371
1372	case BPF_STX | BPF_ATOMIC | BPF_W: {
 
1373		const u8 tmp = bpf2sparc[TMP_REG_1];
1374		const u8 tmp2 = bpf2sparc[TMP_REG_2];
1375		const u8 tmp3 = bpf2sparc[TMP_REG_3];
1376
1377		if (insn->imm != BPF_ADD) {
1378			pr_err_once("unknown atomic op %02x\n", insn->imm);
1379			return -EINVAL;
1380		}
1381
1382		/* lock *(u32 *)(dst + off) += src */
1383
1384		if (insn->dst_reg == BPF_REG_FP)
1385			ctx->saw_frame_pointer = true;
1386
1387		ctx->tmp_1_used = true;
1388		ctx->tmp_2_used = true;
1389		ctx->tmp_3_used = true;
1390		emit_loadimm(off, tmp, ctx);
1391		emit_alu3(ADD, dst, tmp, tmp, ctx);
1392
1393		emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1394		emit_alu3(ADD, tmp2, src, tmp3, ctx);
1395		emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1396		emit_cmp(tmp2, tmp3, ctx);
1397		emit_branch(BNE, 4, 0, ctx);
1398		emit_nop(ctx);
1399		break;
1400	}
1401	/* STX XADD: lock *(u64 *)(dst + off) += src */
1402	case BPF_STX | BPF_ATOMIC | BPF_DW: {
1403		const u8 tmp = bpf2sparc[TMP_REG_1];
1404		const u8 tmp2 = bpf2sparc[TMP_REG_2];
1405		const u8 tmp3 = bpf2sparc[TMP_REG_3];
1406
1407		if (insn->imm != BPF_ADD) {
1408			pr_err_once("unknown atomic op %02x\n", insn->imm);
1409			return -EINVAL;
1410		}
1411
1412		if (insn->dst_reg == BPF_REG_FP)
1413			ctx->saw_frame_pointer = true;
1414
1415		ctx->tmp_1_used = true;
1416		ctx->tmp_2_used = true;
1417		ctx->tmp_3_used = true;
1418		emit_loadimm(off, tmp, ctx);
1419		emit_alu3(ADD, dst, tmp, tmp, ctx);
1420
1421		emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1422		emit_alu3(ADD, tmp2, src, tmp3, ctx);
1423		emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1424		emit_cmp(tmp2, tmp3, ctx);
1425		emit_branch(BNE, 4, 0, ctx);
1426		emit_nop(ctx);
1427		break;
1428	}
1429
1430	default:
1431		pr_err_once("unknown opcode %02x\n", code);
1432		return -EINVAL;
1433	}
1434
1435	return 0;
1436}
1437
1438static int build_body(struct jit_ctx *ctx)
1439{
1440	const struct bpf_prog *prog = ctx->prog;
1441	int i;
1442
1443	for (i = 0; i < prog->len; i++) {
1444		const struct bpf_insn *insn = &prog->insnsi[i];
1445		int ret;
1446
1447		ret = build_insn(insn, ctx);
1448
1449		if (ret > 0) {
1450			i++;
1451			ctx->offset[i] = ctx->idx;
1452			continue;
1453		}
1454		ctx->offset[i] = ctx->idx;
1455		if (ret)
1456			return ret;
1457	}
1458	return 0;
1459}
1460
1461static void jit_fill_hole(void *area, unsigned int size)
1462{
1463	u32 *ptr;
1464	/* We are guaranteed to have aligned memory. */
1465	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1466		*ptr++ = 0x91d02005; /* ta 5 */
1467}
1468
1469bool bpf_jit_needs_zext(void)
1470{
1471	return true;
1472}
1473
1474struct sparc64_jit_data {
1475	struct bpf_binary_header *header;
1476	u8 *image;
1477	struct jit_ctx ctx;
1478};
1479
1480struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1481{
1482	struct bpf_prog *tmp, *orig_prog = prog;
1483	struct sparc64_jit_data *jit_data;
1484	struct bpf_binary_header *header;
1485	u32 prev_image_size, image_size;
1486	bool tmp_blinded = false;
1487	bool extra_pass = false;
1488	struct jit_ctx ctx;
1489	u8 *image_ptr;
1490	int pass, i;
1491
1492	if (!prog->jit_requested)
1493		return orig_prog;
1494
1495	tmp = bpf_jit_blind_constants(prog);
1496	/* If blinding was requested and we failed during blinding,
1497	 * we must fall back to the interpreter.
1498	 */
1499	if (IS_ERR(tmp))
1500		return orig_prog;
1501	if (tmp != prog) {
1502		tmp_blinded = true;
1503		prog = tmp;
1504	}
1505
1506	jit_data = prog->aux->jit_data;
1507	if (!jit_data) {
1508		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1509		if (!jit_data) {
1510			prog = orig_prog;
1511			goto out;
1512		}
1513		prog->aux->jit_data = jit_data;
1514	}
1515	if (jit_data->ctx.offset) {
1516		ctx = jit_data->ctx;
1517		image_ptr = jit_data->image;
1518		header = jit_data->header;
1519		extra_pass = true;
1520		image_size = sizeof(u32) * ctx.idx;
1521		prev_image_size = image_size;
1522		pass = 1;
1523		goto skip_init_ctx;
1524	}
1525
1526	memset(&ctx, 0, sizeof(ctx));
1527	ctx.prog = prog;
1528
1529	ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
1530	if (ctx.offset == NULL) {
1531		prog = orig_prog;
1532		goto out_off;
1533	}
1534
1535	/* Longest sequence emitted is for bswap32, 12 instructions.  Pre-cook
1536	 * the offset array so that we converge faster.
1537	 */
1538	for (i = 0; i < prog->len; i++)
1539		ctx.offset[i] = i * (12 * 4);
1540
1541	prev_image_size = ~0U;
1542	for (pass = 1; pass < 40; pass++) {
1543		ctx.idx = 0;
1544
1545		build_prologue(&ctx);
1546		if (build_body(&ctx)) {
1547			prog = orig_prog;
1548			goto out_off;
1549		}
1550		build_epilogue(&ctx);
1551
1552		if (bpf_jit_enable > 1)
1553			pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
1554				ctx.idx * 4,
1555				ctx.tmp_1_used ? '1' : ' ',
1556				ctx.tmp_2_used ? '2' : ' ',
1557				ctx.tmp_3_used ? '3' : ' ',
1558				ctx.saw_frame_pointer ? 'F' : ' ',
1559				ctx.saw_call ? 'C' : ' ',
1560				ctx.saw_tail_call ? 'T' : ' ');
1561
1562		if (ctx.idx * 4 == prev_image_size)
1563			break;
1564		prev_image_size = ctx.idx * 4;
1565		cond_resched();
1566	}
1567
1568	/* Now we know the actual image size. */
1569	image_size = sizeof(u32) * ctx.idx;
1570	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1571				      sizeof(u32), jit_fill_hole);
1572	if (header == NULL) {
1573		prog = orig_prog;
1574		goto out_off;
1575	}
1576
1577	ctx.image = (u32 *)image_ptr;
1578skip_init_ctx:
1579	ctx.idx = 0;
1580
1581	build_prologue(&ctx);
1582
1583	if (build_body(&ctx)) {
1584		bpf_jit_binary_free(header);
1585		prog = orig_prog;
1586		goto out_off;
1587	}
1588
1589	build_epilogue(&ctx);
1590
1591	if (ctx.idx * 4 != prev_image_size) {
1592		pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1593		       prev_image_size, ctx.idx * 4);
1594		bpf_jit_binary_free(header);
1595		prog = orig_prog;
1596		goto out_off;
1597	}
1598
1599	if (bpf_jit_enable > 1)
1600		bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1601
1602	bpf_flush_icache(header, (u8 *)header + header->size);
1603
1604	if (!prog->is_func || extra_pass) {
1605		bpf_jit_binary_lock_ro(header);
1606	} else {
1607		jit_data->ctx = ctx;
1608		jit_data->image = image_ptr;
1609		jit_data->header = header;
1610	}
1611
1612	prog->bpf_func = (void *)ctx.image;
1613	prog->jited = 1;
1614	prog->jited_len = image_size;
1615
1616	if (!prog->is_func || extra_pass) {
1617		bpf_prog_fill_jited_linfo(prog, ctx.offset);
1618out_off:
1619		kfree(ctx.offset);
1620		kfree(jit_data);
1621		prog->aux->jit_data = NULL;
1622	}
1623out:
1624	if (tmp_blinded)
1625		bpf_jit_prog_release_other(prog, prog == orig_prog ?
1626					   tmp : orig_prog);
1627	return prog;
1628}