1/*
  2 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
  3 *
  4 * This program is free software; you can redistribute it and/or modify
  5 * it under the terms of the GNU General Public License as published by
  6 * the Free Software Foundation; either version 2 of the License, or
  7 * (at your option) any later version.
  8 */
  9
 10#include <linux/module.h>
 11#include <linux/init.h>
 12#include <linux/kernel.h>
 13#include <linux/skbuff.h>
 14#include <linux/rtnetlink.h>
 15#include <linux/filter.h>
 16#include <linux/bpf.h>
 17
 18#include <net/netlink.h>
 19#include <net/pkt_sched.h>
 20
 21#include <linux/tc_act/tc_bpf.h>
 22#include <net/tc_act/tc_bpf.h>
 23
 24#define ACT_BPF_NAME_LEN	256
 25
 26struct tcf_bpf_cfg {
 27	struct bpf_prog *filter;
 28	struct sock_filter *bpf_ops;
 29	const char *bpf_name;
 30	u16 bpf_num_ops;
 31	bool is_ebpf;
 32};
 33
 34static unsigned int bpf_net_id;
 35static struct tc_action_ops act_bpf_ops;
 36
 37static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act,
 38		   struct tcf_result *res)
 39{
 40	bool at_ingress = skb_at_tc_ingress(skb);
 41	struct tcf_bpf *prog = to_bpf(act);
 42	struct bpf_prog *filter;
 43	int action, filter_res;
 44
 45	tcf_lastuse_update(&prog->tcf_tm);
 46	bstats_cpu_update(this_cpu_ptr(prog->common.cpu_bstats), skb);
 47
 48	rcu_read_lock();
 49	filter = rcu_dereference(prog->filter);
 50	if (at_ingress) {
 51		__skb_push(skb, skb->mac_len);
 52		bpf_compute_data_pointers(skb);
 53		filter_res = BPF_PROG_RUN(filter, skb);
 54		__skb_pull(skb, skb->mac_len);
 55	} else {
 56		bpf_compute_data_pointers(skb);
 57		filter_res = BPF_PROG_RUN(filter, skb);
 58	}
 59	rcu_read_unlock();
 60
 61	/* A BPF program may overwrite the default action opcode.
 62	 * Similarly as in cls_bpf, if filter_res == -1 we use the
 63	 * default action specified from tc.
 64	 *
 65	 * In case a different well-known TC_ACT opcode has been
 66	 * returned, it will overwrite the default one.
 67	 *
 68	 * For everything else that is unkown, TC_ACT_UNSPEC is
 69	 * returned.
 70	 */
 71	switch (filter_res) {
 72	case TC_ACT_PIPE:
 73	case TC_ACT_RECLASSIFY:
 74	case TC_ACT_OK:
 75	case TC_ACT_REDIRECT:
 76		action = filter_res;
 77		break;
 78	case TC_ACT_SHOT:
 79		action = filter_res;
 80		qstats_drop_inc(this_cpu_ptr(prog->common.cpu_qstats));
 81		break;
 82	case TC_ACT_UNSPEC:
 83		action = prog->tcf_action;
 84		break;
 85	default:
 86		action = TC_ACT_UNSPEC;
 87		break;
 88	}
 89
 90	return action;
 91}
 92
 93static bool tcf_bpf_is_ebpf(const struct tcf_bpf *prog)
 94{
 95	return !prog->bpf_ops;
 96}
 97
 98static int tcf_bpf_dump_bpf_info(const struct tcf_bpf *prog,
 99				 struct sk_buff *skb)
100{
101	struct nlattr *nla;
102
103	if (nla_put_u16(skb, TCA_ACT_BPF_OPS_LEN, prog->bpf_num_ops))
104		return -EMSGSIZE;
105
106	nla = nla_reserve(skb, TCA_ACT_BPF_OPS, prog->bpf_num_ops *
107			  sizeof(struct sock_filter));
108	if (nla == NULL)
109		return -EMSGSIZE;
110
111	memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));
112
113	return 0;
114}
115
116static int tcf_bpf_dump_ebpf_info(const struct tcf_bpf *prog,
117				  struct sk_buff *skb)
118{
119	struct nlattr *nla;
120
121	if (prog->bpf_name &&
122	    nla_put_string(skb, TCA_ACT_BPF_NAME, prog->bpf_name))
123		return -EMSGSIZE;
124
125	if (nla_put_u32(skb, TCA_ACT_BPF_ID, prog->filter->aux->id))
126		return -EMSGSIZE;
127
128	nla = nla_reserve(skb, TCA_ACT_BPF_TAG, sizeof(prog->filter->tag));
129	if (nla == NULL)
130		return -EMSGSIZE;
131
132	memcpy(nla_data(nla), prog->filter->tag, nla_len(nla));
133
134	return 0;
135}
136
137static int tcf_bpf_dump(struct sk_buff *skb, struct tc_action *act,
138			int bind, int ref)
139{
140	unsigned char *tp = skb_tail_pointer(skb);
141	struct tcf_bpf *prog = to_bpf(act);
142	struct tc_act_bpf opt = {
143		.index   = prog->tcf_index,
144		.refcnt  = prog->tcf_refcnt - ref,
145		.bindcnt = prog->tcf_bindcnt - bind,
146		.action  = prog->tcf_action,
147	};
148	struct tcf_t tm;
149	int ret;
150
151	if (nla_put(skb, TCA_ACT_BPF_PARMS, sizeof(opt), &opt))
152		goto nla_put_failure;
153
154	if (tcf_bpf_is_ebpf(prog))
155		ret = tcf_bpf_dump_ebpf_info(prog, skb);
156	else
157		ret = tcf_bpf_dump_bpf_info(prog, skb);
158	if (ret)
159		goto nla_put_failure;
160
161	tcf_tm_dump(&tm, &prog->tcf_tm);
162	if (nla_put_64bit(skb, TCA_ACT_BPF_TM, sizeof(tm), &tm,
163			  TCA_ACT_BPF_PAD))
164		goto nla_put_failure;
165
166	return skb->len;
167
168nla_put_failure:
169	nlmsg_trim(skb, tp);
170	return -1;
171}
172
173static const struct nla_policy act_bpf_policy[TCA_ACT_BPF_MAX + 1] = {
174	[TCA_ACT_BPF_PARMS]	= { .len = sizeof(struct tc_act_bpf) },
175	[TCA_ACT_BPF_FD]	= { .type = NLA_U32 },
176	[TCA_ACT_BPF_NAME]	= { .type = NLA_NUL_STRING,
177				    .len = ACT_BPF_NAME_LEN },
178	[TCA_ACT_BPF_OPS_LEN]	= { .type = NLA_U16 },
179	[TCA_ACT_BPF_OPS]	= { .type = NLA_BINARY,
180				    .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
181};
182
183static int tcf_bpf_init_from_ops(struct nlattr **tb, struct tcf_bpf_cfg *cfg)
184{
185	struct sock_filter *bpf_ops;
186	struct sock_fprog_kern fprog_tmp;
187	struct bpf_prog *fp;
188	u16 bpf_size, bpf_num_ops;
189	int ret;
190
191	bpf_num_ops = nla_get_u16(tb[TCA_ACT_BPF_OPS_LEN]);
192	if (bpf_num_ops	> BPF_MAXINSNS || bpf_num_ops == 0)
193		return -EINVAL;
194
195	bpf_size = bpf_num_ops * sizeof(*bpf_ops);
196	if (bpf_size != nla_len(tb[TCA_ACT_BPF_OPS]))
197		return -EINVAL;
198
199	bpf_ops = kzalloc(bpf_size, GFP_KERNEL);
200	if (bpf_ops == NULL)
201		return -ENOMEM;
202
203	memcpy(bpf_ops, nla_data(tb[TCA_ACT_BPF_OPS]), bpf_size);
204
205	fprog_tmp.len = bpf_num_ops;
206	fprog_tmp.filter = bpf_ops;
207
208	ret = bpf_prog_create(&fp, &fprog_tmp);
209	if (ret < 0) {
210		kfree(bpf_ops);
211		return ret;
212	}
213
214	cfg->bpf_ops = bpf_ops;
215	cfg->bpf_num_ops = bpf_num_ops;
216	cfg->filter = fp;
217	cfg->is_ebpf = false;
218
219	return 0;
220}
221
222static int tcf_bpf_init_from_efd(struct nlattr **tb, struct tcf_bpf_cfg *cfg)
223{
224	struct bpf_prog *fp;
225	char *name = NULL;
226	u32 bpf_fd;
227
228	bpf_fd = nla_get_u32(tb[TCA_ACT_BPF_FD]);
229
230	fp = bpf_prog_get_type(bpf_fd, BPF_PROG_TYPE_SCHED_ACT);
231	if (IS_ERR(fp))
232		return PTR_ERR(fp);
233
234	if (tb[TCA_ACT_BPF_NAME]) {
235		name = nla_memdup(tb[TCA_ACT_BPF_NAME], GFP_KERNEL);
236		if (!name) {
237			bpf_prog_put(fp);
238			return -ENOMEM;
239		}
240	}
241
242	cfg->bpf_name = name;
243	cfg->filter = fp;
244	cfg->is_ebpf = true;
245
246	return 0;
247}
248
249static void tcf_bpf_cfg_cleanup(const struct tcf_bpf_cfg *cfg)
250{
251	struct bpf_prog *filter = cfg->filter;
252
253	if (filter) {
254		if (cfg->is_ebpf)
255			bpf_prog_put(filter);
256		else
257			bpf_prog_destroy(filter);
258	}
259
260	kfree(cfg->bpf_ops);
261	kfree(cfg->bpf_name);
262}
263
264static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf *prog,
265				  struct tcf_bpf_cfg *cfg)
266{
267	cfg->is_ebpf = tcf_bpf_is_ebpf(prog);
268	/* updates to prog->filter are prevented, since it's called either
269	 * with rtnl lock or during final cleanup in rcu callback
270	 */
271	cfg->filter = rcu_dereference_protected(prog->filter, 1);
272
273	cfg->bpf_ops = prog->bpf_ops;
274	cfg->bpf_name = prog->bpf_name;
275}
276
277static int tcf_bpf_init(struct net *net, struct nlattr *nla,
278			struct nlattr *est, struct tc_action **act,
279			int replace, int bind, struct netlink_ext_ack *extack)
280{
281	struct tc_action_net *tn = net_generic(net, bpf_net_id);
282	struct nlattr *tb[TCA_ACT_BPF_MAX + 1];
283	struct tcf_bpf_cfg cfg, old;
284	struct tc_act_bpf *parm;
285	struct tcf_bpf *prog;
286	bool is_bpf, is_ebpf;
287	int ret, res = 0;
288
289	if (!nla)
290		return -EINVAL;
291
292	ret = nla_parse_nested(tb, TCA_ACT_BPF_MAX, nla, act_bpf_policy, NULL);
293	if (ret < 0)
294		return ret;
295
296	if (!tb[TCA_ACT_BPF_PARMS])
297		return -EINVAL;
298
299	parm = nla_data(tb[TCA_ACT_BPF_PARMS]);
300
301	if (!tcf_idr_check(tn, parm->index, act, bind)) {
302		ret = tcf_idr_create(tn, parm->index, est, act,
303				     &act_bpf_ops, bind, true);
304		if (ret < 0)
305			return ret;
306
307		res = ACT_P_CREATED;
308	} else {
309		/* Don't override defaults. */
310		if (bind)
311			return 0;
312
313		tcf_idr_release(*act, bind);
314		if (!replace)
315			return -EEXIST;
316	}
317
318	is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS];
319	is_ebpf = tb[TCA_ACT_BPF_FD];
320
321	if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) {
322		ret = -EINVAL;
323		goto out;
324	}
325
326	memset(&cfg, 0, sizeof(cfg));
327
328	ret = is_bpf ? tcf_bpf_init_from_ops(tb, &cfg) :
329		       tcf_bpf_init_from_efd(tb, &cfg);
330	if (ret < 0)
331		goto out;
332
333	prog = to_bpf(*act);
334	ASSERT_RTNL();
335
336	if (res != ACT_P_CREATED)
337		tcf_bpf_prog_fill_cfg(prog, &old);
338
339	prog->bpf_ops = cfg.bpf_ops;
340	prog->bpf_name = cfg.bpf_name;
341
342	if (cfg.bpf_num_ops)
343		prog->bpf_num_ops = cfg.bpf_num_ops;
344
345	prog->tcf_action = parm->action;
346	rcu_assign_pointer(prog->filter, cfg.filter);
347
348	if (res == ACT_P_CREATED) {
349		tcf_idr_insert(tn, *act);
350	} else {
351		/* make sure the program being replaced is no longer executing */
352		synchronize_rcu();
353		tcf_bpf_cfg_cleanup(&old);
354	}
355
356	return res;
357out:
358	if (res == ACT_P_CREATED)
359		tcf_idr_release(*act, bind);
360
361	return ret;
362}
363
364static void tcf_bpf_cleanup(struct tc_action *act)
365{
366	struct tcf_bpf_cfg tmp;
367
368	tcf_bpf_prog_fill_cfg(to_bpf(act), &tmp);
369	tcf_bpf_cfg_cleanup(&tmp);
370}
371
372static int tcf_bpf_walker(struct net *net, struct sk_buff *skb,
373			  struct netlink_callback *cb, int type,
374			  const struct tc_action_ops *ops,
375			  struct netlink_ext_ack *extack)
376{
377	struct tc_action_net *tn = net_generic(net, bpf_net_id);
378
379	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
380}
381
382static int tcf_bpf_search(struct net *net, struct tc_action **a, u32 index,
383			  struct netlink_ext_ack *extack)
384{
385	struct tc_action_net *tn = net_generic(net, bpf_net_id);
386
387	return tcf_idr_search(tn, a, index);
388}
389
390static struct tc_action_ops act_bpf_ops __read_mostly = {
391	.kind		=	"bpf",
392	.type		=	TCA_ACT_BPF,
393	.owner		=	THIS_MODULE,
394	.act		=	tcf_bpf,
395	.dump		=	tcf_bpf_dump,
396	.cleanup	=	tcf_bpf_cleanup,
397	.init		=	tcf_bpf_init,
398	.walk		=	tcf_bpf_walker,
399	.lookup		=	tcf_bpf_search,
400	.size		=	sizeof(struct tcf_bpf),
401};
402
403static __net_init int bpf_init_net(struct net *net)
404{
405	struct tc_action_net *tn = net_generic(net, bpf_net_id);
406
407	return tc_action_net_init(tn, &act_bpf_ops);
408}
409
410static void __net_exit bpf_exit_net(struct list_head *net_list)
411{
412	tc_action_net_exit(net_list, bpf_net_id);
413}
414
415static struct pernet_operations bpf_net_ops = {
416	.init = bpf_init_net,
417	.exit_batch = bpf_exit_net,
418	.id   = &bpf_net_id,
419	.size = sizeof(struct tc_action_net),
420};
421
422static int __init bpf_init_module(void)
423{
424	return tcf_register_action(&act_bpf_ops, &bpf_net_ops);
425}
426
427static void __exit bpf_cleanup_module(void)
428{
429	tcf_unregister_action(&act_bpf_ops, &bpf_net_ops);
430}
431
432module_init(bpf_init_module);
433module_exit(bpf_cleanup_module);
434
435MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
436MODULE_DESCRIPTION("TC BPF based action");
437MODULE_LICENSE("GPL v2");