Linux Audio

Check our new training course

Loading...
v6.8
  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * eBPF JIT compiler
  4 *
  5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
  6 *		  IBM Corporation
  7 *
  8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
 
  9 */
 10#include <linux/moduleloader.h>
 11#include <asm/cacheflush.h>
 12#include <asm/asm-compat.h>
 13#include <linux/netdevice.h>
 14#include <linux/filter.h>
 15#include <linux/if_vlan.h>
 16#include <linux/kernel.h>
 17#include <linux/memory.h>
 18#include <linux/bpf.h>
 19
 20#include <asm/kprobes.h>
 21#include <asm/code-patching.h>
 22
 23#include "bpf_jit.h"
 24
 25static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
 26{
 27	memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
 
 28}
 29
 30int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
 
 31{
 32	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
 33		PPC_JMP(exit_addr);
 34	} else if (ctx->alt_exit_addr) {
 35		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
 36			return -1;
 37		PPC_JMP(ctx->alt_exit_addr);
 38	} else {
 39		ctx->alt_exit_addr = ctx->idx * 4;
 40		bpf_jit_build_epilogue(image, ctx);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 41	}
 42
 43	return 0;
 44}
 
 
 
 
 
 
 
 
 
 
 
 45
 46struct powerpc_jit_data {
 47	/* address of rw header */
 48	struct bpf_binary_header *hdr;
 49	/* address of ro final header */
 50	struct bpf_binary_header *fhdr;
 51	u32 *addrs;
 52	u8 *fimage;
 53	u32 proglen;
 54	struct codegen_context ctx;
 55};
 56
 57bool bpf_jit_needs_zext(void)
 58{
 59	return true;
 60}
 61
 62struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 63{
 64	u32 proglen;
 65	u32 alloclen;
 66	u8 *image = NULL;
 67	u32 *code_base;
 68	u32 *addrs;
 69	struct powerpc_jit_data *jit_data;
 70	struct codegen_context cgctx;
 71	int pass;
 72	int flen;
 73	struct bpf_binary_header *fhdr = NULL;
 74	struct bpf_binary_header *hdr = NULL;
 75	struct bpf_prog *org_fp = fp;
 76	struct bpf_prog *tmp_fp;
 77	bool bpf_blinded = false;
 78	bool extra_pass = false;
 79	u8 *fimage = NULL;
 80	u32 *fcode_base;
 81	u32 extable_len;
 82	u32 fixup_len;
 83
 84	if (!fp->jit_requested)
 85		return org_fp;
 86
 87	tmp_fp = bpf_jit_blind_constants(org_fp);
 88	if (IS_ERR(tmp_fp))
 89		return org_fp;
 90
 91	if (tmp_fp != org_fp) {
 92		bpf_blinded = true;
 93		fp = tmp_fp;
 94	}
 95
 96	jit_data = fp->aux->jit_data;
 97	if (!jit_data) {
 98		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
 99		if (!jit_data) {
100			fp = org_fp;
101			goto out;
 
 
 
 
 
 
 
 
102		}
103		fp->aux->jit_data = jit_data;
104	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
106	flen = fp->len;
107	addrs = jit_data->addrs;
108	if (addrs) {
109		cgctx = jit_data->ctx;
110		/*
111		 * JIT compiled to a writable location (image/code_base) first.
112		 * It is then moved to the readonly final location (fimage/fcode_base)
113		 * using instruction patching.
114		 */
115		fimage = jit_data->fimage;
116		fhdr = jit_data->fhdr;
117		proglen = jit_data->proglen;
118		hdr = jit_data->hdr;
119		image = (void *)hdr + ((void *)fimage - (void *)fhdr);
120		extra_pass = true;
121		/* During extra pass, ensure index is reset before repopulating extable entries */
122		cgctx.exentry_idx = 0;
123		goto skip_init_ctx;
124	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
126	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
127	if (addrs == NULL) {
128		fp = org_fp;
129		goto out_addrs;
130	}
 
 
131
132	memset(&cgctx, 0, sizeof(struct codegen_context));
133	bpf_jit_init_reg_mapping(&cgctx);
 
 
 
 
 
 
 
 
 
 
 
134
135	/* Make sure that the stack is quadword aligned. */
136	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
137
138	/* Scouting faux-generate pass 0 */
139	if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
140		/* We hit something illegal or unsupported. */
141		fp = org_fp;
142		goto out_addrs;
143	}
144
145	/*
146	 * If we have seen a tail call, we need a second pass.
147	 * This is because bpf_jit_emit_common_epilogue() is called
148	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
149	 * We also need a second pass if we ended up with too large
150	 * a program so as to ensure BPF_EXIT branches are in range.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151	 */
152	if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
153		cgctx.idx = 0;
154		if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
155			fp = org_fp;
156			goto out_addrs;
157		}
158	}
159
160	bpf_jit_realloc_regs(&cgctx);
 
 
 
 
 
 
 
161	/*
162	 * Pretend to build prologue, given the features we've seen.  This will
163	 * update ctgtx.idx as it pretends to output instructions, then we can
164	 * calculate total size from idx.
165	 */
166	bpf_jit_build_prologue(NULL, &cgctx);
167	addrs[fp->len] = cgctx.idx * 4;
168	bpf_jit_build_epilogue(NULL, &cgctx);
169
170	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
171	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
172
173	proglen = cgctx.idx * 4;
174	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
175
176	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
177					      bpf_jit_fill_ill_insns);
178	if (!fhdr) {
179		fp = org_fp;
180		goto out_addrs;
181	}
182
183	if (extable_len)
184		fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
185
186skip_init_ctx:
187	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
188	fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
189
190	/* Code generation passes 1-2 */
191	for (pass = 1; pass < 3; pass++) {
192		/* Now build the prologue, body code & epilogue for real. */
193		cgctx.idx = 0;
194		cgctx.alt_exit_addr = 0;
195		bpf_jit_build_prologue(code_base, &cgctx);
196		if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
197				       extra_pass)) {
198			bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
199			bpf_jit_binary_pack_free(fhdr, hdr);
200			fp = org_fp;
201			goto out_addrs;
202		}
203		bpf_jit_build_epilogue(code_base, &cgctx);
204
205		if (bpf_jit_enable > 1)
206			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
207				proglen - (cgctx.idx * 4), cgctx.seen);
208	}
209
210	if (bpf_jit_enable > 1)
211		/*
212		 * Note that we output the base address of the code_base
213		 * rather than image, since opcodes are in code_base.
214		 */
215		bpf_jit_dump(flen, proglen, pass, code_base);
216
217#ifdef CONFIG_PPC64_ELF_ABI_V1
 
 
218	/* Function descriptor nastiness: Address + TOC */
219	((u64 *)image)[0] = (u64)fcode_base;
220	((u64 *)image)[1] = local_paca->kernel_toc;
221#endif
222
223	fp->bpf_func = (void *)fimage;
224	fp->jited = 1;
225	fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
226
227	if (!fp->is_func || extra_pass) {
228		if (bpf_jit_binary_pack_finalize(fp, fhdr, hdr)) {
229			fp = org_fp;
230			goto out_addrs;
231		}
232		bpf_prog_fill_jited_linfo(fp, addrs);
233out_addrs:
234		kfree(addrs);
235		kfree(jit_data);
236		fp->aux->jit_data = NULL;
237	} else {
238		jit_data->addrs = addrs;
239		jit_data->ctx = cgctx;
240		jit_data->proglen = proglen;
241		jit_data->fimage = fimage;
242		jit_data->fhdr = fhdr;
243		jit_data->hdr = hdr;
244	}
245
246out:
247	if (bpf_blinded)
248		bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
249
250	return fp;
251}
252
253/*
254 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
255 * this function, as this only applies to BPF_PROBE_MEM, for now.
256 */
257int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
258			  struct codegen_context *ctx, int insn_idx, int jmp_off,
259			  int dst_reg)
260{
261	off_t offset;
262	unsigned long pc;
263	struct exception_table_entry *ex, *ex_entry;
264	u32 *fixup;
265
266	/* Populate extable entries only in the last pass */
267	if (pass != 2)
268		return 0;
269
270	if (!fp->aux->extable ||
271	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
272		return -EINVAL;
273
274	/*
275	 * Program is first written to image before copying to the
276	 * final location (fimage). Accordingly, update in the image first.
277	 * As all offsets used are relative, copying as is to the
278	 * final location should be alright.
279	 */
280	pc = (unsigned long)&image[insn_idx];
281	ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
282
283	fixup = (void *)ex -
284		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
285		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);
286
287	fixup[0] = PPC_RAW_LI(dst_reg, 0);
288	if (IS_ENABLED(CONFIG_PPC32))
289		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
290
291	fixup[BPF_FIXUP_LEN - 1] =
292		PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
293
294	ex_entry = &ex[ctx->exentry_idx];
295
296	offset = pc - (long)&ex_entry->insn;
297	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
298		return -ERANGE;
299	ex_entry->insn = offset;
300
301	offset = (long)fixup - (long)&ex_entry->fixup;
302	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
303		return -ERANGE;
304	ex_entry->fixup = offset;
305
306	ctx->exentry_idx++;
307	return 0;
308}
309
310void *bpf_arch_text_copy(void *dst, void *src, size_t len)
311{
312	int err;
313
314	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
315		return ERR_PTR(-EINVAL);
316
317	mutex_lock(&text_mutex);
318	err = patch_instructions(dst, src, len, false);
319	mutex_unlock(&text_mutex);
320
321	return err ? ERR_PTR(err) : dst;
322}
323
324int bpf_arch_text_invalidate(void *dst, size_t len)
325{
326	u32 insn = BREAKPOINT_INSTRUCTION;
327	int ret;
328
329	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
330		return -EINVAL;
331
332	mutex_lock(&text_mutex);
333	ret = patch_instructions(dst, &insn, len, true);
334	mutex_unlock(&text_mutex);
335
336	return ret;
337}
338
339void bpf_jit_free(struct bpf_prog *fp)
340{
341	if (fp->jited) {
342		struct powerpc_jit_data *jit_data = fp->aux->jit_data;
343		struct bpf_binary_header *hdr;
344
345		/*
346		 * If we fail the final pass of JIT (from jit_subprogs),
347		 * the program may not be finalized yet. Call finalize here
348		 * before freeing it.
349		 */
350		if (jit_data) {
351			bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr);
352			kvfree(jit_data->addrs);
353			kfree(jit_data);
354		}
355		hdr = bpf_jit_binary_pack_hdr(fp);
356		bpf_jit_binary_pack_free(hdr, NULL);
357		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
358	}
359
360	bpf_prog_unlock_free(fp);
361}
v5.9
  1// SPDX-License-Identifier: GPL-2.0-only
  2/* bpf_jit_comp.c: BPF JIT compiler
 
  3 *
  4 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
 
  5 *
  6 * Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com)
  7 * Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org>
  8 */
  9#include <linux/moduleloader.h>
 10#include <asm/cacheflush.h>
 11#include <asm/asm-compat.h>
 12#include <linux/netdevice.h>
 13#include <linux/filter.h>
 14#include <linux/if_vlan.h>
 
 
 
 15
 16#include "bpf_jit32.h"
 
 17
 18static inline void bpf_flush_icache(void *start, void *end)
 
 
 19{
 20	smp_wmb();
 21	flush_icache_range((unsigned long)start, (unsigned long)end);
 22}
 23
 24static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image,
 25				   struct codegen_context *ctx)
 26{
 27	int i;
 28	const struct sock_filter *filter = fp->insns;
 29
 30	if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) {
 31		/* Make stackframe */
 32		if (ctx->seen & SEEN_DATAREF) {
 33			/* If we call any helpers (for loads), save LR */
 34			EMIT(PPC_INST_MFLR | __PPC_RT(R0));
 35			PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
 36
 37			/* Back up non-volatile regs. */
 38			PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D)));
 39			PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL)));
 40		}
 41		if (ctx->seen & SEEN_MEM) {
 42			/*
 43			 * Conditionally save regs r15-r31 as some will be used
 44			 * for M[] data.
 45			 */
 46			for (i = r_M; i < (r_M+16); i++) {
 47				if (ctx->seen & (1 << (i-r_M)))
 48					PPC_BPF_STL(i, 1, -(REG_SZ*(32-i)));
 49			}
 50		}
 51		PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME);
 52	}
 53
 54	if (ctx->seen & SEEN_DATAREF) {
 55		/*
 56		 * If this filter needs to access skb data,
 57		 * prepare r_D and r_HL:
 58		 *  r_HL = skb->len - skb->data_len
 59		 *  r_D	 = skb->data
 60		 */
 61		PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
 62							 data_len));
 63		PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len));
 64		EMIT(PPC_RAW_SUB(r_HL, r_HL, r_scratch1));
 65		PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data));
 66	}
 67
 68	if (ctx->seen & SEEN_XREG) {
 69		/*
 70		 * TODO: Could also detect whether first instr. sets X and
 71		 * avoid this (as below, with A).
 72		 */
 73		EMIT(PPC_RAW_LI(r_X, 0));
 74	}
 
 
 
 75
 76	/* make sure we dont leak kernel information to user */
 77	if (bpf_needs_clear_a(&filter[0]))
 78		EMIT(PPC_RAW_LI(r_A, 0));
 79}
 80
 81static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
 82{
 83	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 84
 85	if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) {
 86		EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME));
 87		if (ctx->seen & SEEN_DATAREF) {
 88			PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
 89			EMIT(PPC_RAW_MTLR(0));
 90			PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D)));
 91			PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL)));
 92		}
 93		if (ctx->seen & SEEN_MEM) {
 94			/* Restore any saved non-vol registers */
 95			for (i = r_M; i < (r_M+16); i++) {
 96				if (ctx->seen & (1 << (i-r_M)))
 97					PPC_BPF_LL(i, 1, -(REG_SZ*(32-i)));
 98			}
 99		}
 
100	}
101	/* The RETs have left a return value in R3. */
102
103	EMIT(PPC_RAW_BLR());
104}
105
106#define CHOOSE_LOAD_FUNC(K, func) \
107	((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
108
109/* Assemble the body code between the prologue & epilogue. */
110static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
111			      struct codegen_context *ctx,
112			      unsigned int *addrs)
113{
114	const struct sock_filter *filter = fp->insns;
115	int flen = fp->len;
116	u8 *func;
117	unsigned int true_cond;
118	int i;
119
120	/* Start of epilogue code */
121	unsigned int exit_addr = addrs[flen];
122
123	for (i = 0; i < flen; i++) {
124		unsigned int K = filter[i].k;
125		u16 code = bpf_anc_helper(&filter[i]);
126
 
 
 
 
127		/*
128		 * addrs[] maps a BPF bytecode address into a real offset from
129		 * the start of the body code.
 
130		 */
131		addrs[i] = ctx->idx * 4;
132
133		switch (code) {
134			/*** ALU ops ***/
135		case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */
136			ctx->seen |= SEEN_XREG;
137			EMIT(PPC_RAW_ADD(r_A, r_A, r_X));
138			break;
139		case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */
140			if (!K)
141				break;
142			EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(K)));
143			if (K >= 32768)
144				EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(K)));
145			break;
146		case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */
147			ctx->seen |= SEEN_XREG;
148			EMIT(PPC_RAW_SUB(r_A, r_A, r_X));
149			break;
150		case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */
151			if (!K)
152				break;
153			EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(-K)));
154			if (K >= 32768)
155				EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(-K)));
156			break;
157		case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */
158			ctx->seen |= SEEN_XREG;
159			EMIT(PPC_RAW_MULW(r_A, r_A, r_X));
160			break;
161		case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */
162			if (K < 32768)
163				EMIT(PPC_RAW_MULI(r_A, r_A, K));
164			else {
165				PPC_LI32(r_scratch1, K);
166				EMIT(PPC_RAW_MULW(r_A, r_A, r_scratch1));
167			}
168			break;
169		case BPF_ALU | BPF_MOD | BPF_X: /* A %= X; */
170		case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */
171			ctx->seen |= SEEN_XREG;
172			EMIT(PPC_RAW_CMPWI(r_X, 0));
173			if (ctx->pc_ret0 != -1) {
174				PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
175			} else {
176				PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
177				EMIT(PPC_RAW_LI(r_ret, 0));
178				PPC_JMP(exit_addr);
179			}
180			if (code == (BPF_ALU | BPF_MOD | BPF_X)) {
181				EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_X));
182				EMIT(PPC_RAW_MULW(r_scratch1, r_X, r_scratch1));
183				EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
184			} else {
185				EMIT(PPC_RAW_DIVWU(r_A, r_A, r_X));
186			}
187			break;
188		case BPF_ALU | BPF_MOD | BPF_K: /* A %= K; */
189			PPC_LI32(r_scratch2, K);
190			EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_scratch2));
191			EMIT(PPC_RAW_MULW(r_scratch1, r_scratch2, r_scratch1));
192			EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
193			break;
194		case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */
195			if (K == 1)
196				break;
197			PPC_LI32(r_scratch1, K);
198			EMIT(PPC_RAW_DIVWU(r_A, r_A, r_scratch1));
199			break;
200		case BPF_ALU | BPF_AND | BPF_X:
201			ctx->seen |= SEEN_XREG;
202			EMIT(PPC_RAW_AND(r_A, r_A, r_X));
203			break;
204		case BPF_ALU | BPF_AND | BPF_K:
205			if (!IMM_H(K))
206				EMIT(PPC_RAW_ANDI(r_A, r_A, K));
207			else {
208				PPC_LI32(r_scratch1, K);
209				EMIT(PPC_RAW_AND(r_A, r_A, r_scratch1));
210			}
211			break;
212		case BPF_ALU | BPF_OR | BPF_X:
213			ctx->seen |= SEEN_XREG;
214			EMIT(PPC_RAW_OR(r_A, r_A, r_X));
215			break;
216		case BPF_ALU | BPF_OR | BPF_K:
217			if (IMM_L(K))
218				EMIT(PPC_RAW_ORI(r_A, r_A, IMM_L(K)));
219			if (K >= 65536)
220				EMIT(PPC_RAW_ORIS(r_A, r_A, IMM_H(K)));
221			break;
222		case BPF_ANC | SKF_AD_ALU_XOR_X:
223		case BPF_ALU | BPF_XOR | BPF_X: /* A ^= X */
224			ctx->seen |= SEEN_XREG;
225			EMIT(PPC_RAW_XOR(r_A, r_A, r_X));
226			break;
227		case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */
228			if (IMM_L(K))
229				EMIT(PPC_RAW_XORI(r_A, r_A, IMM_L(K)));
230			if (K >= 65536)
231				EMIT(PPC_RAW_XORIS(r_A, r_A, IMM_H(K)));
232			break;
233		case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */
234			ctx->seen |= SEEN_XREG;
235			EMIT(PPC_RAW_SLW(r_A, r_A, r_X));
236			break;
237		case BPF_ALU | BPF_LSH | BPF_K:
238			if (K == 0)
239				break;
240			else
241				EMIT(PPC_RAW_SLWI(r_A, r_A, K));
242			break;
243		case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */
244			ctx->seen |= SEEN_XREG;
245			EMIT(PPC_RAW_SRW(r_A, r_A, r_X));
246			break;
247		case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */
248			if (K == 0)
249				break;
250			else
251				EMIT(PPC_RAW_SRWI(r_A, r_A, K));
252			break;
253		case BPF_ALU | BPF_NEG:
254			EMIT(PPC_RAW_NEG(r_A, r_A));
255			break;
256		case BPF_RET | BPF_K:
257			PPC_LI32(r_ret, K);
258			if (!K) {
259				if (ctx->pc_ret0 == -1)
260					ctx->pc_ret0 = i;
261			}
262			/*
263			 * If this isn't the very last instruction, branch to
264			 * the epilogue if we've stuff to clean up.  Otherwise,
265			 * if there's nothing to tidy, just return.  If we /are/
266			 * the last instruction, we're about to fall through to
267			 * the epilogue to return.
268			 */
269			if (i != flen - 1) {
270				/*
271				 * Note: 'seen' is properly valid only on pass
272				 * #2.	Both parts of this conditional are the
273				 * same instruction size though, meaning the
274				 * first pass will still correctly determine the
275				 * code size/addresses.
276				 */
277				if (ctx->seen)
278					PPC_JMP(exit_addr);
279				else
280					EMIT(PPC_RAW_BLR());
281			}
282			break;
283		case BPF_RET | BPF_A:
284			EMIT(PPC_RAW_MR(r_ret, r_A));
285			if (i != flen - 1) {
286				if (ctx->seen)
287					PPC_JMP(exit_addr);
288				else
289					EMIT(PPC_RAW_BLR());
290			}
291			break;
292		case BPF_MISC | BPF_TAX: /* X = A */
293			EMIT(PPC_RAW_MR(r_X, r_A));
294			break;
295		case BPF_MISC | BPF_TXA: /* A = X */
296			ctx->seen |= SEEN_XREG;
297			EMIT(PPC_RAW_MR(r_A, r_X));
298			break;
299
300			/*** Constant loads/M[] access ***/
301		case BPF_LD | BPF_IMM: /* A = K */
302			PPC_LI32(r_A, K);
303			break;
304		case BPF_LDX | BPF_IMM: /* X = K */
305			PPC_LI32(r_X, K);
306			break;
307		case BPF_LD | BPF_MEM: /* A = mem[K] */
308			EMIT(PPC_RAW_MR(r_A, r_M + (K & 0xf)));
309			ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
310			break;
311		case BPF_LDX | BPF_MEM: /* X = mem[K] */
312			EMIT(PPC_RAW_MR(r_X, r_M + (K & 0xf)));
313			ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
314			break;
315		case BPF_ST: /* mem[K] = A */
316			EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_A));
317			ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
318			break;
319		case BPF_STX: /* mem[K] = X */
320			EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_X));
321			ctx->seen |= SEEN_XREG | SEEN_MEM | (1<<(K & 0xf));
322			break;
323		case BPF_LD | BPF_W | BPF_LEN: /*	A = skb->len; */
324			BUILD_BUG_ON(sizeof_field(struct sk_buff, len) != 4);
325			PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
326			break;
327		case BPF_LDX | BPF_W | BPF_ABS: /* A = *((u32 *)(seccomp_data + K)); */
328			PPC_LWZ_OFFS(r_A, r_skb, K);
329			break;
330		case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */
331			PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
332			break;
333
334			/*** Ancillary info loads ***/
335		case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
336			BUILD_BUG_ON(sizeof_field(struct sk_buff,
337						  protocol) != 2);
338			PPC_NTOHS_OFFS(r_A, r_skb, offsetof(struct sk_buff,
339							    protocol));
340			break;
341		case BPF_ANC | SKF_AD_IFINDEX:
342		case BPF_ANC | SKF_AD_HATYPE:
343			BUILD_BUG_ON(sizeof_field(struct net_device,
344						ifindex) != 4);
345			BUILD_BUG_ON(sizeof_field(struct net_device,
346						type) != 2);
347			PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
348								dev));
349			EMIT(PPC_RAW_CMPDI(r_scratch1, 0));
350			if (ctx->pc_ret0 != -1) {
351				PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
352			} else {
353				/* Exit, returning 0; first pass hits here. */
354				PPC_BCC_SHORT(COND_NE, ctx->idx * 4 + 12);
355				EMIT(PPC_RAW_LI(r_ret, 0));
356				PPC_JMP(exit_addr);
357			}
358			if (code == (BPF_ANC | SKF_AD_IFINDEX)) {
359				PPC_LWZ_OFFS(r_A, r_scratch1,
360				     offsetof(struct net_device, ifindex));
361			} else {
362				PPC_LHZ_OFFS(r_A, r_scratch1,
363				     offsetof(struct net_device, type));
364			}
365
366			break;
367		case BPF_ANC | SKF_AD_MARK:
368			BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
369			PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
370							  mark));
371			break;
372		case BPF_ANC | SKF_AD_RXHASH:
373			BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
374			PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
375							  hash));
376			break;
377		case BPF_ANC | SKF_AD_VLAN_TAG:
378			BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
379
380			PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
381							  vlan_tci));
382			break;
383		case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
384			PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET());
385			if (PKT_VLAN_PRESENT_BIT)
386				EMIT(PPC_RAW_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT));
387			if (PKT_VLAN_PRESENT_BIT < 7)
388				EMIT(PPC_RAW_ANDI(r_A, r_A, 1));
389			break;
390		case BPF_ANC | SKF_AD_QUEUE:
391			BUILD_BUG_ON(sizeof_field(struct sk_buff,
392						  queue_mapping) != 2);
393			PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
394							  queue_mapping));
395			break;
396		case BPF_ANC | SKF_AD_PKTTYPE:
397			PPC_LBZ_OFFS(r_A, r_skb, PKT_TYPE_OFFSET());
398			EMIT(PPC_RAW_ANDI(r_A, r_A, PKT_TYPE_MAX));
399			EMIT(PPC_RAW_SRWI(r_A, r_A, 5));
400			break;
401		case BPF_ANC | SKF_AD_CPU:
402			PPC_BPF_LOAD_CPU(r_A);
403			break;
404			/*** Absolute loads from packet header/data ***/
405		case BPF_LD | BPF_W | BPF_ABS:
406			func = CHOOSE_LOAD_FUNC(K, sk_load_word);
407			goto common_load;
408		case BPF_LD | BPF_H | BPF_ABS:
409			func = CHOOSE_LOAD_FUNC(K, sk_load_half);
410			goto common_load;
411		case BPF_LD | BPF_B | BPF_ABS:
412			func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
413		common_load:
414			/* Load from [K]. */
415			ctx->seen |= SEEN_DATAREF;
416			PPC_FUNC_ADDR(r_scratch1, func);
417			EMIT(PPC_RAW_MTLR(r_scratch1));
418			PPC_LI32(r_addr, K);
419			EMIT(PPC_RAW_BLRL());
420			/*
421			 * Helper returns 'lt' condition on error, and an
422			 * appropriate return value in r3
423			 */
424			PPC_BCC(COND_LT, exit_addr);
425			break;
426
427			/*** Indirect loads from packet header/data ***/
428		case BPF_LD | BPF_W | BPF_IND:
429			func = sk_load_word;
430			goto common_load_ind;
431		case BPF_LD | BPF_H | BPF_IND:
432			func = sk_load_half;
433			goto common_load_ind;
434		case BPF_LD | BPF_B | BPF_IND:
435			func = sk_load_byte;
436		common_load_ind:
437			/*
438			 * Load from [X + K].  Negative offsets are tested for
439			 * in the helper functions.
440			 */
441			ctx->seen |= SEEN_DATAREF | SEEN_XREG;
442			PPC_FUNC_ADDR(r_scratch1, func);
443			EMIT(PPC_RAW_MTLR(r_scratch1));
444			EMIT(PPC_RAW_ADDI(r_addr, r_X, IMM_L(K)));
445			if (K >= 32768)
446				EMIT(PPC_RAW_ADDIS(r_addr, r_addr, IMM_HA(K)));
447			EMIT(PPC_RAW_BLRL());
448			/* If error, cr0.LT set */
449			PPC_BCC(COND_LT, exit_addr);
450			break;
451
452		case BPF_LDX | BPF_B | BPF_MSH:
453			func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
454			goto common_load;
455			break;
456
457			/*** Jump and branches ***/
458		case BPF_JMP | BPF_JA:
459			if (K != 0)
460				PPC_JMP(addrs[i + 1 + K]);
461			break;
462
463		case BPF_JMP | BPF_JGT | BPF_K:
464		case BPF_JMP | BPF_JGT | BPF_X:
465			true_cond = COND_GT;
466			goto cond_branch;
467		case BPF_JMP | BPF_JGE | BPF_K:
468		case BPF_JMP | BPF_JGE | BPF_X:
469			true_cond = COND_GE;
470			goto cond_branch;
471		case BPF_JMP | BPF_JEQ | BPF_K:
472		case BPF_JMP | BPF_JEQ | BPF_X:
473			true_cond = COND_EQ;
474			goto cond_branch;
475		case BPF_JMP | BPF_JSET | BPF_K:
476		case BPF_JMP | BPF_JSET | BPF_X:
477			true_cond = COND_NE;
478		cond_branch:
479			/* same targets, can avoid doing the test :) */
480			if (filter[i].jt == filter[i].jf) {
481				if (filter[i].jt > 0)
482					PPC_JMP(addrs[i + 1 + filter[i].jt]);
483				break;
484			}
485
486			switch (code) {
487			case BPF_JMP | BPF_JGT | BPF_X:
488			case BPF_JMP | BPF_JGE | BPF_X:
489			case BPF_JMP | BPF_JEQ | BPF_X:
490				ctx->seen |= SEEN_XREG;
491				EMIT(PPC_RAW_CMPLW(r_A, r_X));
492				break;
493			case BPF_JMP | BPF_JSET | BPF_X:
494				ctx->seen |= SEEN_XREG;
495				EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, r_X));
496				break;
497			case BPF_JMP | BPF_JEQ | BPF_K:
498			case BPF_JMP | BPF_JGT | BPF_K:
499			case BPF_JMP | BPF_JGE | BPF_K:
500				if (K < 32768)
501					EMIT(PPC_RAW_CMPLWI(r_A, K));
502				else {
503					PPC_LI32(r_scratch1, K);
504					EMIT(PPC_RAW_CMPLW(r_A, r_scratch1));
505				}
506				break;
507			case BPF_JMP | BPF_JSET | BPF_K:
508				if (K < 32768)
509					/* PPC_ANDI is /only/ dot-form */
510					EMIT(PPC_RAW_ANDI(r_scratch1, r_A, K));
511				else {
512					PPC_LI32(r_scratch1, K);
513					EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A,
514						    r_scratch1));
515				}
516				break;
517			}
518			/* Sometimes branches are constructed "backward", with
519			 * the false path being the branch and true path being
520			 * a fallthrough to the next instruction.
521			 */
522			if (filter[i].jt == 0)
523				/* Swap the sense of the branch */
524				PPC_BCC(true_cond ^ COND_CMP_TRUE,
525					addrs[i + 1 + filter[i].jf]);
526			else {
527				PPC_BCC(true_cond, addrs[i + 1 + filter[i].jt]);
528				if (filter[i].jf != 0)
529					PPC_JMP(addrs[i + 1 + filter[i].jf]);
530			}
531			break;
532		default:
533			/* The filter contains something cruel & unusual.
534			 * We don't handle it, but also there shouldn't be
535			 * anything missing from our list.
536			 */
537			if (printk_ratelimit())
538				pr_err("BPF filter opcode %04x (@%d) unsupported\n",
539				       filter[i].code, i);
540			return -ENOTSUPP;
541		}
542
 
 
 
 
543	}
544	/* Set end-of-body-code address for exit. */
545	addrs[i] = ctx->idx * 4;
546
547	return 0;
548}
549
550void bpf_jit_compile(struct bpf_prog *fp)
551{
552	unsigned int proglen;
553	unsigned int alloclen;
554	u32 *image = NULL;
555	u32 *code_base;
556	unsigned int *addrs;
557	struct codegen_context cgctx;
558	int pass;
559	int flen = fp->len;
560
561	if (!bpf_jit_enable)
562		return;
563
564	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
565	if (addrs == NULL)
566		return;
 
 
 
567
568	/*
569	 * There are multiple assembly passes as the generated code will change
570	 * size as it settles down, figuring out the max branch offsets/exit
571	 * paths required.
572	 *
573	 * The range of standard conditional branches is +/- 32Kbytes.	Since
574	 * BPF_MAXINSNS = 4096, we can only jump from (worst case) start to
575	 * finish with 8 bytes/instruction.  Not feasible, so long jumps are
576	 * used, distinct from short branches.
577	 *
578	 * Current:
579	 *
580	 * For now, both branch types assemble to 2 words (short branches padded
581	 * with a NOP); this is less efficient, but assembly will always complete
582	 * after exactly 3 passes:
583	 *
584	 * First pass: No code buffer; Program is "faux-generated" -- no code
585	 * emitted but maximum size of output determined (and addrs[] filled
586	 * in).	 Also, we note whether we use M[], whether we use skb data, etc.
587	 * All generation choices assumed to be 'worst-case', e.g. branches all
588	 * far (2 instructions), return path code reduction not available, etc.
589	 *
590	 * Second pass: Code buffer allocated with size determined previously.
591	 * Prologue generated to support features we have seen used.  Exit paths
592	 * determined and addrs[] is filled in again, as code may be slightly
593	 * smaller as a result.
594	 *
595	 * Third pass: Code generated 'for real', and branch destinations
596	 * determined from now-accurate addrs[] map.
597	 *
598	 * Ideal:
599	 *
600	 * If we optimise this, near branches will be shorter.	On the
601	 * first assembly pass, we should err on the side of caution and
602	 * generate the biggest code.  On subsequent passes, branches will be
603	 * generated short or long and code size will reduce.  With smaller
604	 * code, more branches may fall into the short category, and code will
605	 * reduce more.
606	 *
607	 * Finally, if we see one pass generate code the same size as the
608	 * previous pass we have converged and should now generate code for
609	 * real.  Allocating at the end will also save the memory that would
610	 * otherwise be wasted by the (small) current code shrinkage.
611	 * Preferably, we should do a small number of passes (e.g. 5) and if we
612	 * haven't converged by then, get impatient and force code to generate
613	 * as-is, even if the odd branch would be left long.  The chances of a
614	 * long jump are tiny with all but the most enormous of BPF filter
615	 * inputs, so we should usually converge on the third pass.
616	 */
 
 
 
 
 
 
 
617
618	cgctx.idx = 0;
619	cgctx.seen = 0;
620	cgctx.pc_ret0 = -1;
621	/* Scouting faux-generate pass 0 */
622	if (bpf_jit_build_body(fp, 0, &cgctx, addrs))
623		/* We hit something illegal or unsupported. */
624		goto out;
625
626	/*
627	 * Pretend to build prologue, given the features we've seen.  This will
628	 * update ctgtx.idx as it pretends to output instructions, then we can
629	 * calculate total size from idx.
630	 */
631	bpf_jit_build_prologue(fp, 0, &cgctx);
632	bpf_jit_build_epilogue(0, &cgctx);
 
 
 
 
633
634	proglen = cgctx.idx * 4;
635	alloclen = proglen + FUNCTION_DESCR_SIZE;
636	image = module_alloc(alloclen);
637	if (!image)
638		goto out;
 
 
 
 
639
640	code_base = image + (FUNCTION_DESCR_SIZE/4);
 
 
 
 
 
641
642	/* Code generation passes 1-2 */
643	for (pass = 1; pass < 3; pass++) {
644		/* Now build the prologue, body code & epilogue for real. */
645		cgctx.idx = 0;
646		bpf_jit_build_prologue(fp, code_base, &cgctx);
647		bpf_jit_build_body(fp, code_base, &cgctx, addrs);
 
 
 
 
 
 
 
648		bpf_jit_build_epilogue(code_base, &cgctx);
649
650		if (bpf_jit_enable > 1)
651			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
652				proglen - (cgctx.idx * 4), cgctx.seen);
653	}
654
655	if (bpf_jit_enable > 1)
656		/* Note that we output the base address of the code_base
 
657		 * rather than image, since opcodes are in code_base.
658		 */
659		bpf_jit_dump(flen, proglen, pass, code_base);
660
661	bpf_flush_icache(code_base, code_base + (proglen/4));
662
663#ifdef CONFIG_PPC64
664	/* Function descriptor nastiness: Address + TOC */
665	((u64 *)image)[0] = (u64)code_base;
666	((u64 *)image)[1] = local_paca->kernel_toc;
667#endif
668
669	fp->bpf_func = (void *)image;
670	fp->jited = 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
671
672out:
673	kfree(addrs);
674	return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
675}
676
677void bpf_jit_free(struct bpf_prog *fp)
678{
679	if (fp->jited)
680		module_memfree(fp->bpf_func);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
681
682	bpf_prog_unlock_free(fp);
683}