1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2019 Intel Corporation. */
  3
  4#include <linux/hash.h>
  5#include <linux/bpf.h>
  6#include <linux/filter.h>
  7
  8/* The BPF dispatcher is a multiway branch code generator. The
  9 * dispatcher is a mechanism to avoid the performance penalty of an
 10 * indirect call, which is expensive when retpolines are enabled. A
 11 * dispatch client registers a BPF program into the dispatcher, and if
 12 * there is available room in the dispatcher a direct call to the BPF
 13 * program will be generated. All calls to the BPF programs called via
 14 * the dispatcher will then be a direct call, instead of an
 15 * indirect. The dispatcher hijacks a trampoline function it via the
 16 * __fentry__ of the trampoline. The trampoline function has the
 17 * following signature:
 18 *
 19 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
 20 *                         unsigned int (*bpf_func)(const void *,
 21 *                                                  const struct bpf_insn *));
 22 */
 23
 24static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
 25	struct bpf_dispatcher *d, struct bpf_prog *prog)
 26{
 27	int i;
 28
 29	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
 30		if (prog == d->progs[i].prog)
 31			return &d->progs[i];
 32	}
 33	return NULL;
 34}
 35
 36static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
 37	struct bpf_dispatcher *d)
 38{
 39	return bpf_dispatcher_find_prog(d, NULL);
 40}
 41
 42static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
 43				    struct bpf_prog *prog)
 44{
 45	struct bpf_dispatcher_prog *entry;
 46
 47	if (!prog)
 48		return false;
 49
 50	entry = bpf_dispatcher_find_prog(d, prog);
 51	if (entry) {
 52		refcount_inc(&entry->users);
 53		return false;
 54	}
 55
 56	entry = bpf_dispatcher_find_free(d);
 57	if (!entry)
 58		return false;
 59
 60	bpf_prog_inc(prog);
 61	entry->prog = prog;
 62	refcount_set(&entry->users, 1);
 63	d->num_progs++;
 64	return true;
 65}
 66
 67static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
 68				       struct bpf_prog *prog)
 69{
 70	struct bpf_dispatcher_prog *entry;
 71
 72	if (!prog)
 73		return false;
 74
 75	entry = bpf_dispatcher_find_prog(d, prog);
 76	if (!entry)
 77		return false;
 78
 79	if (refcount_dec_and_test(&entry->users)) {
 80		entry->prog = NULL;
 81		bpf_prog_put(prog);
 82		d->num_progs--;
 83		return true;
 84	}
 85	return false;
 86}
 87
 88int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
 89{
 90	return -ENOTSUPP;
 91}
 92
 93static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
 94{
 95	s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
 96	int i;
 97
 98	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
 99		if (d->progs[i].prog)
100			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
101	}
102	return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
103}
104
105static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
106{
107	void *old, *new;
108	u32 noff;
109	int err;
110
111	if (!prev_num_progs) {
112		old = NULL;
113		noff = 0;
114	} else {
115		old = d->image + d->image_off;
116		noff = d->image_off ^ (PAGE_SIZE / 2);
117	}
118
119	new = d->num_progs ? d->image + noff : NULL;
120	if (new) {
121		if (bpf_dispatcher_prepare(d, new))
122			return;
123	}
124
125	err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
126	if (err || !new)
127		return;
128
129	d->image_off = noff;
130}
131
132void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
133				struct bpf_prog *to)
134{
135	bool changed = false;
136	int prev_num_progs;
137
138	if (from == to)
139		return;
140
141	mutex_lock(&d->mutex);
142	if (!d->image) {
143		d->image = bpf_jit_alloc_exec_page();
144		if (!d->image)
145			goto out;
146		bpf_image_ksym_add(d->image, &d->ksym);
147	}
148
149	prev_num_progs = d->num_progs;
150	changed |= bpf_dispatcher_remove_prog(d, from);
151	changed |= bpf_dispatcher_add_prog(d, to);
152
153	if (!changed)
154		goto out;
155
156	bpf_dispatcher_update(d, prev_num_progs);
157out:
158	mutex_unlock(&d->mutex);
159}