1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2019 Intel Corporation. */
  3
  4#include <linux/hash.h>
  5#include <linux/bpf.h>
  6#include <linux/filter.h>
  7#include <linux/static_call.h>
  8
  9/* The BPF dispatcher is a multiway branch code generator. The
 10 * dispatcher is a mechanism to avoid the performance penalty of an
 11 * indirect call, which is expensive when retpolines are enabled. A
 12 * dispatch client registers a BPF program into the dispatcher, and if
 13 * there is available room in the dispatcher a direct call to the BPF
 14 * program will be generated. All calls to the BPF programs called via
 15 * the dispatcher will then be a direct call, instead of an
 16 * indirect. The dispatcher hijacks a trampoline function it via the
 17 * __fentry__ of the trampoline. The trampoline function has the
 18 * following signature:
 19 *
 20 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
 21 *                         unsigned int (*bpf_func)(const void *,
 22 *                                                  const struct bpf_insn *));
 23 */
 24
 25static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
 26	struct bpf_dispatcher *d, struct bpf_prog *prog)
 27{
 28	int i;
 29
 30	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
 31		if (prog == d->progs[i].prog)
 32			return &d->progs[i];
 33	}
 34	return NULL;
 35}
 36
 37static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
 38	struct bpf_dispatcher *d)
 39{
 40	return bpf_dispatcher_find_prog(d, NULL);
 41}
 42
 43static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
 44				    struct bpf_prog *prog)
 45{
 46	struct bpf_dispatcher_prog *entry;
 47
 48	if (!prog)
 49		return false;
 50
 51	entry = bpf_dispatcher_find_prog(d, prog);
 52	if (entry) {
 53		refcount_inc(&entry->users);
 54		return false;
 55	}
 56
 57	entry = bpf_dispatcher_find_free(d);
 58	if (!entry)
 59		return false;
 60
 61	bpf_prog_inc(prog);
 62	entry->prog = prog;
 63	refcount_set(&entry->users, 1);
 64	d->num_progs++;
 65	return true;
 66}
 67
 68static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
 69				       struct bpf_prog *prog)
 70{
 71	struct bpf_dispatcher_prog *entry;
 72
 73	if (!prog)
 74		return false;
 75
 76	entry = bpf_dispatcher_find_prog(d, prog);
 77	if (!entry)
 78		return false;
 79
 80	if (refcount_dec_and_test(&entry->users)) {
 81		entry->prog = NULL;
 82		bpf_prog_put(prog);
 83		d->num_progs--;
 84		return true;
 85	}
 86	return false;
 87}
 88
 89int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
 90{
 91	return -ENOTSUPP;
 92}
 93
 94static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
 95{
 96	s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
 97	int i;
 98
 99	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
100		if (d->progs[i].prog)
101			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
102	}
103	return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
104}
105
106static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
107{
108	void *new, *tmp;
109	u32 noff = 0;
110
111	if (prev_num_progs)
112		noff = d->image_off ^ (PAGE_SIZE / 2);
113
114	new = d->num_progs ? d->image + noff : NULL;
115	tmp = d->num_progs ? d->rw_image + noff : NULL;
116	if (new) {
117		/* Prepare the dispatcher in d->rw_image. Then use
118		 * bpf_arch_text_copy to update d->image, which is RO+X.
119		 */
120		if (bpf_dispatcher_prepare(d, new, tmp))
121			return;
122		if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
123			return;
124	}
125
126	__BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);
127
128	/* Make sure all the callers executing the previous/old half of the
129	 * image leave it, so following update call can modify it safely.
130	 */
131	synchronize_rcu();
132
133	if (new)
134		d->image_off = noff;
135}
136
137void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
138				struct bpf_prog *to)
139{
140	bool changed = false;
141	int prev_num_progs;
142
143	if (from == to)
144		return;
145
146	mutex_lock(&d->mutex);
147	if (!d->image) {
148		d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
149		if (!d->image)
150			goto out;
151		d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
152		if (!d->rw_image) {
153			u32 size = PAGE_SIZE;
154
155			bpf_arch_text_copy(d->image, &size, sizeof(size));
156			bpf_prog_pack_free((struct bpf_binary_header *)d->image);
157			d->image = NULL;
158			goto out;
159		}
160		bpf_image_ksym_add(d->image, &d->ksym);
161	}
162
163	prev_num_progs = d->num_progs;
164	changed |= bpf_dispatcher_remove_prog(d, from);
165	changed |= bpf_dispatcher_add_prog(d, to);
166
167	if (!changed)
168		goto out;
169
170	bpf_dispatcher_update(d, prev_num_progs);
171out:
172	mutex_unlock(&d->mutex);
173}