1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2017 Facebook
  3 */
  4#include <linux/bpf.h>
  5#include <linux/slab.h>
  6#include <linux/vmalloc.h>
  7#include <linux/etherdevice.h>
  8#include <linux/filter.h>
  9#include <linux/sched/signal.h>
 10#include <net/bpf_sk_storage.h>
 11#include <net/sock.h>
 12#include <net/tcp.h>
 13
 14#define CREATE_TRACE_POINTS
 15#include <trace/events/bpf_test_run.h>
 16
 17static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
 18			u32 *retval, u32 *time)
 19{
 20	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
 21	enum bpf_cgroup_storage_type stype;
 22	u64 time_start, time_spent = 0;
 23	int ret = 0;
 24	u32 i;
 25
 26	for_each_cgroup_storage_type(stype) {
 27		storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
 28		if (IS_ERR(storage[stype])) {
 29			storage[stype] = NULL;
 30			for_each_cgroup_storage_type(stype)
 31				bpf_cgroup_storage_free(storage[stype]);
 32			return -ENOMEM;
 33		}
 34	}
 35
 36	if (!repeat)
 37		repeat = 1;
 38
 39	rcu_read_lock();
 40	preempt_disable();
 41	time_start = ktime_get_ns();
 42	for (i = 0; i < repeat; i++) {
 43		bpf_cgroup_storage_set(storage);
 44		*retval = BPF_PROG_RUN(prog, ctx);
 45
 46		if (signal_pending(current)) {
 47			ret = -EINTR;
 48			break;
 49		}
 50
 51		if (need_resched()) {
 52			time_spent += ktime_get_ns() - time_start;
 53			preempt_enable();
 54			rcu_read_unlock();
 55
 56			cond_resched();
 57
 58			rcu_read_lock();
 59			preempt_disable();
 60			time_start = ktime_get_ns();
 61		}
 62	}
 63	time_spent += ktime_get_ns() - time_start;
 64	preempt_enable();
 65	rcu_read_unlock();
 66
 67	do_div(time_spent, repeat);
 68	*time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
 69
 70	for_each_cgroup_storage_type(stype)
 71		bpf_cgroup_storage_free(storage[stype]);
 72
 73	return ret;
 74}
 75
 76static int bpf_test_finish(const union bpf_attr *kattr,
 77			   union bpf_attr __user *uattr, const void *data,
 78			   u32 size, u32 retval, u32 duration)
 79{
 80	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
 81	int err = -EFAULT;
 82	u32 copy_size = size;
 83
 84	/* Clamp copy if the user has provided a size hint, but copy the full
 85	 * buffer if not to retain old behaviour.
 86	 */
 87	if (kattr->test.data_size_out &&
 88	    copy_size > kattr->test.data_size_out) {
 89		copy_size = kattr->test.data_size_out;
 90		err = -ENOSPC;
 91	}
 92
 93	if (data_out && copy_to_user(data_out, data, copy_size))
 94		goto out;
 95	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
 96		goto out;
 97	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
 98		goto out;
 99	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
100		goto out;
101	if (err != -ENOSPC)
102		err = 0;
103out:
104	trace_bpf_test_finish(&err);
105	return err;
106}
107
108static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
109			   u32 headroom, u32 tailroom)
110{
111	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
112	void *data;
113
114	if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
115		return ERR_PTR(-EINVAL);
116
117	data = kzalloc(size + headroom + tailroom, GFP_USER);
118	if (!data)
119		return ERR_PTR(-ENOMEM);
120
121	if (copy_from_user(data + headroom, data_in, size)) {
122		kfree(data);
123		return ERR_PTR(-EFAULT);
124	}
125	return data;
126}
127
128static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
129{
130	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
131	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
132	u32 size = kattr->test.ctx_size_in;
133	void *data;
134	int err;
135
136	if (!data_in && !data_out)
137		return NULL;
138
139	data = kzalloc(max_size, GFP_USER);
140	if (!data)
141		return ERR_PTR(-ENOMEM);
142
143	if (data_in) {
144		err = bpf_check_uarg_tail_zero(data_in, max_size, size);
145		if (err) {
146			kfree(data);
147			return ERR_PTR(err);
148		}
149
150		size = min_t(u32, max_size, size);
151		if (copy_from_user(data, data_in, size)) {
152			kfree(data);
153			return ERR_PTR(-EFAULT);
154		}
155	}
156	return data;
157}
158
159static int bpf_ctx_finish(const union bpf_attr *kattr,
160			  union bpf_attr __user *uattr, const void *data,
161			  u32 size)
162{
163	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
164	int err = -EFAULT;
165	u32 copy_size = size;
166
167	if (!data || !data_out)
168		return 0;
169
170	if (copy_size > kattr->test.ctx_size_out) {
171		copy_size = kattr->test.ctx_size_out;
172		err = -ENOSPC;
173	}
174
175	if (copy_to_user(data_out, data, copy_size))
176		goto out;
177	if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
178		goto out;
179	if (err != -ENOSPC)
180		err = 0;
181out:
182	return err;
183}
184
185/**
186 * range_is_zero - test whether buffer is initialized
187 * @buf: buffer to check
188 * @from: check from this position
189 * @to: check up until (excluding) this position
190 *
191 * This function returns true if the there is a non-zero byte
192 * in the buf in the range [from,to).
193 */
194static inline bool range_is_zero(void *buf, size_t from, size_t to)
195{
196	return !memchr_inv((u8 *)buf + from, 0, to - from);
197}
198
199static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
200{
201	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
202
203	if (!__skb)
204		return 0;
205
206	/* make sure the fields we don't use are zeroed */
207	if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority)))
208		return -EINVAL;
209
210	/* priority is allowed */
211
212	if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) +
213			   FIELD_SIZEOF(struct __sk_buff, priority),
214			   offsetof(struct __sk_buff, cb)))
215		return -EINVAL;
216
217	/* cb is allowed */
218
219	if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
220			   FIELD_SIZEOF(struct __sk_buff, cb),
221			   sizeof(struct __sk_buff)))
222		return -EINVAL;
223
224	skb->priority = __skb->priority;
225	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
226
227	return 0;
228}
229
230static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
231{
232	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
233
234	if (!__skb)
235		return;
236
237	__skb->priority = skb->priority;
238	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
239}
240
241int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
242			  union bpf_attr __user *uattr)
243{
244	bool is_l2 = false, is_direct_pkt_access = false;
245	u32 size = kattr->test.data_size_in;
246	u32 repeat = kattr->test.repeat;
247	struct __sk_buff *ctx = NULL;
248	u32 retval, duration;
249	int hh_len = ETH_HLEN;
250	struct sk_buff *skb;
251	struct sock *sk;
252	void *data;
253	int ret;
254
255	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
256			     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
257	if (IS_ERR(data))
258		return PTR_ERR(data);
259
260	ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
261	if (IS_ERR(ctx)) {
262		kfree(data);
263		return PTR_ERR(ctx);
264	}
265
266	switch (prog->type) {
267	case BPF_PROG_TYPE_SCHED_CLS:
268	case BPF_PROG_TYPE_SCHED_ACT:
269		is_l2 = true;
270		/* fall through */
271	case BPF_PROG_TYPE_LWT_IN:
272	case BPF_PROG_TYPE_LWT_OUT:
273	case BPF_PROG_TYPE_LWT_XMIT:
274		is_direct_pkt_access = true;
275		break;
276	default:
277		break;
278	}
279
280	sk = kzalloc(sizeof(struct sock), GFP_USER);
281	if (!sk) {
282		kfree(data);
283		kfree(ctx);
284		return -ENOMEM;
285	}
286	sock_net_set(sk, current->nsproxy->net_ns);
287	sock_init_data(NULL, sk);
288
289	skb = build_skb(data, 0);
290	if (!skb) {
291		kfree(data);
292		kfree(ctx);
293		kfree(sk);
294		return -ENOMEM;
295	}
296	skb->sk = sk;
297
298	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
299	__skb_put(skb, size);
300	skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
301	skb_reset_network_header(skb);
302
303	if (is_l2)
304		__skb_push(skb, hh_len);
305	if (is_direct_pkt_access)
306		bpf_compute_data_pointers(skb);
307	ret = convert___skb_to_skb(skb, ctx);
308	if (ret)
309		goto out;
310	ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
311	if (ret)
312		goto out;
313	if (!is_l2) {
314		if (skb_headroom(skb) < hh_len) {
315			int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
316
317			if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
318				ret = -ENOMEM;
319				goto out;
320			}
321		}
322		memset(__skb_push(skb, hh_len), 0, hh_len);
323	}
324	convert_skb_to___skb(skb, ctx);
325
326	size = skb->len;
327	/* bpf program can never convert linear skb to non-linear */
328	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
329		size = skb_headlen(skb);
330	ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
331	if (!ret)
332		ret = bpf_ctx_finish(kattr, uattr, ctx,
333				     sizeof(struct __sk_buff));
334out:
335	kfree_skb(skb);
336	bpf_sk_storage_free(sk);
337	kfree(sk);
338	kfree(ctx);
339	return ret;
340}
341
342int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
343			  union bpf_attr __user *uattr)
344{
345	u32 size = kattr->test.data_size_in;
346	u32 repeat = kattr->test.repeat;
347	struct netdev_rx_queue *rxqueue;
348	struct xdp_buff xdp = {};
349	u32 retval, duration;
350	void *data;
351	int ret;
352
353	if (kattr->test.ctx_in || kattr->test.ctx_out)
354		return -EINVAL;
355
356	data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
357	if (IS_ERR(data))
358		return PTR_ERR(data);
359
360	xdp.data_hard_start = data;
361	xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
362	xdp.data_meta = xdp.data;
363	xdp.data_end = xdp.data + size;
364
365	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
366	xdp.rxq = &rxqueue->xdp_rxq;
367
368	ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
369	if (ret)
370		goto out;
371	if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
372	    xdp.data_end != xdp.data + size)
373		size = xdp.data_end - xdp.data;
374	ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
375out:
376	kfree(data);
377	return ret;
378}
379
380static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
381{
382	/* make sure the fields we don't use are zeroed */
383	if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
384		return -EINVAL;
385
386	/* flags is allowed */
387
388	if (!range_is_zero(ctx, offsetof(struct bpf_flow_keys, flags) +
389			   FIELD_SIZEOF(struct bpf_flow_keys, flags),
390			   sizeof(struct bpf_flow_keys)))
391		return -EINVAL;
392
393	return 0;
394}
395
396int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
397				     const union bpf_attr *kattr,
398				     union bpf_attr __user *uattr)
399{
400	u32 size = kattr->test.data_size_in;
401	struct bpf_flow_dissector ctx = {};
402	u32 repeat = kattr->test.repeat;
403	struct bpf_flow_keys *user_ctx;
404	struct bpf_flow_keys flow_keys;
405	u64 time_start, time_spent = 0;
406	const struct ethhdr *eth;
407	unsigned int flags = 0;
408	u32 retval, duration;
409	void *data;
410	int ret;
411	u32 i;
412
413	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
414		return -EINVAL;
415
416	if (size < ETH_HLEN)
417		return -EINVAL;
418
419	data = bpf_test_init(kattr, size, 0, 0);
420	if (IS_ERR(data))
421		return PTR_ERR(data);
422
423	eth = (struct ethhdr *)data;
424
425	if (!repeat)
426		repeat = 1;
427
428	user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
429	if (IS_ERR(user_ctx)) {
430		kfree(data);
431		return PTR_ERR(user_ctx);
432	}
433	if (user_ctx) {
434		ret = verify_user_bpf_flow_keys(user_ctx);
435		if (ret)
436			goto out;
437		flags = user_ctx->flags;
438	}
439
440	ctx.flow_keys = &flow_keys;
441	ctx.data = data;
442	ctx.data_end = (__u8 *)data + size;
443
444	rcu_read_lock();
445	preempt_disable();
446	time_start = ktime_get_ns();
447	for (i = 0; i < repeat; i++) {
448		retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
449					  size, flags);
450
451		if (signal_pending(current)) {
452			preempt_enable();
453			rcu_read_unlock();
454
455			ret = -EINTR;
456			goto out;
457		}
458
459		if (need_resched()) {
460			time_spent += ktime_get_ns() - time_start;
461			preempt_enable();
462			rcu_read_unlock();
463
464			cond_resched();
465
466			rcu_read_lock();
467			preempt_disable();
468			time_start = ktime_get_ns();
469		}
470	}
471	time_spent += ktime_get_ns() - time_start;
472	preempt_enable();
473	rcu_read_unlock();
474
475	do_div(time_spent, repeat);
476	duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
477
478	ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
479			      retval, duration);
480	if (!ret)
481		ret = bpf_ctx_finish(kattr, uattr, user_ctx,
482				     sizeof(struct bpf_flow_keys));
483
484out:
485	kfree(user_ctx);
486	kfree(data);
487	return ret;
488}