Loading...
Note: File does not exist in v3.15.
1/*
2 * Copyright (C) 2017 Netronome Systems, Inc.
3 *
4 * This software is licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree.
7 *
8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
14 */
15
16#include <linux/bpf.h>
17#include <linux/bpf_verifier.h>
18#include <linux/debugfs.h>
19#include <linux/kernel.h>
20#include <linux/mutex.h>
21#include <linux/rtnetlink.h>
22#include <net/pkt_cls.h>
23
24#include "netdevsim.h"
25
26#define pr_vlog(env, fmt, ...) \
27 bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)
28
29struct nsim_bpf_bound_prog {
30 struct netdevsim *ns;
31 struct bpf_prog *prog;
32 struct dentry *ddir;
33 const char *state;
34 bool is_loaded;
35 struct list_head l;
36};
37
38#define NSIM_BPF_MAX_KEYS 2
39
40struct nsim_bpf_bound_map {
41 struct netdevsim *ns;
42 struct bpf_offloaded_map *map;
43 struct mutex mutex;
44 struct nsim_map_entry {
45 void *key;
46 void *value;
47 } entry[NSIM_BPF_MAX_KEYS];
48 struct list_head l;
49};
50
51static int nsim_debugfs_bpf_string_read(struct seq_file *file, void *data)
52{
53 const char **str = file->private;
54
55 if (*str)
56 seq_printf(file, "%s\n", *str);
57
58 return 0;
59}
60
61static int nsim_debugfs_bpf_string_open(struct inode *inode, struct file *f)
62{
63 return single_open(f, nsim_debugfs_bpf_string_read, inode->i_private);
64}
65
66static const struct file_operations nsim_bpf_string_fops = {
67 .owner = THIS_MODULE,
68 .open = nsim_debugfs_bpf_string_open,
69 .release = single_release,
70 .read = seq_read,
71 .llseek = seq_lseek
72};
73
74static int
75nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
76{
77 struct nsim_bpf_bound_prog *state;
78
79 state = env->prog->aux->offload->dev_priv;
80 if (state->ns->bpf_bind_verifier_delay && !insn_idx)
81 msleep(state->ns->bpf_bind_verifier_delay);
82
83 if (insn_idx == env->prog->len - 1)
84 pr_vlog(env, "Hello from netdevsim!\n");
85
86 return 0;
87}
88
89static const struct bpf_prog_offload_ops nsim_bpf_analyzer_ops = {
90 .insn_hook = nsim_bpf_verify_insn,
91};
92
93static bool nsim_xdp_offload_active(struct netdevsim *ns)
94{
95 return ns->xdp_prog_mode == XDP_ATTACHED_HW;
96}
97
98static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
99{
100 struct nsim_bpf_bound_prog *state;
101
102 if (!prog || !prog->aux->offload)
103 return;
104
105 state = prog->aux->offload->dev_priv;
106 state->is_loaded = loaded;
107}
108
109static int
110nsim_bpf_offload(struct netdevsim *ns, struct bpf_prog *prog, bool oldprog)
111{
112 nsim_prog_set_loaded(ns->bpf_offloaded, false);
113
114 WARN(!!ns->bpf_offloaded != oldprog,
115 "bad offload state, expected offload %sto be active",
116 oldprog ? "" : "not ");
117 ns->bpf_offloaded = prog;
118 ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
119 nsim_prog_set_loaded(prog, true);
120
121 return 0;
122}
123
124int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
125 void *type_data, void *cb_priv)
126{
127 struct tc_cls_bpf_offload *cls_bpf = type_data;
128 struct bpf_prog *prog = cls_bpf->prog;
129 struct netdevsim *ns = cb_priv;
130 struct bpf_prog *oldprog;
131
132 if (type != TC_SETUP_CLSBPF) {
133 NSIM_EA(cls_bpf->common.extack,
134 "only offload of BPF classifiers supported");
135 return -EOPNOTSUPP;
136 }
137
138 if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
139 return -EOPNOTSUPP;
140
141 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
142 NSIM_EA(cls_bpf->common.extack,
143 "only ETH_P_ALL supported as filter protocol");
144 return -EOPNOTSUPP;
145 }
146
147 if (!ns->bpf_tc_accept) {
148 NSIM_EA(cls_bpf->common.extack,
149 "netdevsim configured to reject BPF TC offload");
150 return -EOPNOTSUPP;
151 }
152 /* Note: progs without skip_sw will probably not be dev bound */
153 if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
154 NSIM_EA(cls_bpf->common.extack,
155 "netdevsim configured to reject unbound programs");
156 return -EOPNOTSUPP;
157 }
158
159 if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
160 return -EOPNOTSUPP;
161
162 oldprog = cls_bpf->oldprog;
163
164 /* Don't remove if oldprog doesn't match driver's state */
165 if (ns->bpf_offloaded != oldprog) {
166 oldprog = NULL;
167 if (!cls_bpf->prog)
168 return 0;
169 if (ns->bpf_offloaded) {
170 NSIM_EA(cls_bpf->common.extack,
171 "driver and netdev offload states mismatch");
172 return -EBUSY;
173 }
174 }
175
176 return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
177}
178
179int nsim_bpf_disable_tc(struct netdevsim *ns)
180{
181 if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
182 return -EBUSY;
183 return 0;
184}
185
186static int nsim_xdp_offload_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
187{
188 if (!nsim_xdp_offload_active(ns) && !bpf->prog)
189 return 0;
190 if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
191 NSIM_EA(bpf->extack, "TC program is already loaded");
192 return -EBUSY;
193 }
194
195 return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
196}
197
198static int nsim_xdp_set_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
199{
200 int err;
201
202 if (ns->xdp_prog && (bpf->flags ^ ns->xdp_flags) & XDP_FLAGS_MODES) {
203 NSIM_EA(bpf->extack, "program loaded with different flags");
204 return -EBUSY;
205 }
206
207 if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
208 NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
209 return -EOPNOTSUPP;
210 }
211 if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
212 NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
213 return -EOPNOTSUPP;
214 }
215
216 if (bpf->command == XDP_SETUP_PROG_HW) {
217 err = nsim_xdp_offload_prog(ns, bpf);
218 if (err)
219 return err;
220 }
221
222 if (ns->xdp_prog)
223 bpf_prog_put(ns->xdp_prog);
224
225 ns->xdp_prog = bpf->prog;
226 ns->xdp_flags = bpf->flags;
227
228 if (!bpf->prog)
229 ns->xdp_prog_mode = XDP_ATTACHED_NONE;
230 else if (bpf->command == XDP_SETUP_PROG)
231 ns->xdp_prog_mode = XDP_ATTACHED_DRV;
232 else
233 ns->xdp_prog_mode = XDP_ATTACHED_HW;
234
235 return 0;
236}
237
238static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
239{
240 struct nsim_bpf_bound_prog *state;
241 char name[16];
242
243 state = kzalloc(sizeof(*state), GFP_KERNEL);
244 if (!state)
245 return -ENOMEM;
246
247 state->ns = ns;
248 state->prog = prog;
249 state->state = "verify";
250
251 /* Program id is not populated yet when we create the state. */
252 sprintf(name, "%u", ns->prog_id_gen++);
253 state->ddir = debugfs_create_dir(name, ns->ddir_bpf_bound_progs);
254 if (IS_ERR_OR_NULL(state->ddir)) {
255 kfree(state);
256 return -ENOMEM;
257 }
258
259 debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
260 debugfs_create_file("state", 0400, state->ddir,
261 &state->state, &nsim_bpf_string_fops);
262 debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
263
264 list_add_tail(&state->l, &ns->bpf_bound_progs);
265
266 prog->aux->offload->dev_priv = state;
267
268 return 0;
269}
270
271static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
272{
273 struct nsim_bpf_bound_prog *state;
274
275 state = prog->aux->offload->dev_priv;
276 WARN(state->is_loaded,
277 "offload state destroyed while program still bound");
278 debugfs_remove_recursive(state->ddir);
279 list_del(&state->l);
280 kfree(state);
281}
282
283static int nsim_setup_prog_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
284{
285 if (bpf->prog && bpf->prog->aux->offload) {
286 NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
287 return -EINVAL;
288 }
289 if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
290 NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
291 return -EINVAL;
292 }
293 if (nsim_xdp_offload_active(ns)) {
294 NSIM_EA(bpf->extack, "xdp offload active, can't load drv prog");
295 return -EBUSY;
296 }
297 return 0;
298}
299
300static int
301nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
302{
303 struct nsim_bpf_bound_prog *state;
304
305 if (!bpf->prog)
306 return 0;
307
308 if (!bpf->prog->aux->offload) {
309 NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
310 return -EINVAL;
311 }
312 if (bpf->prog->aux->offload->netdev != ns->netdev) {
313 NSIM_EA(bpf->extack, "program bound to different dev");
314 return -EINVAL;
315 }
316
317 state = bpf->prog->aux->offload->dev_priv;
318 if (WARN_ON(strcmp(state->state, "xlated"))) {
319 NSIM_EA(bpf->extack, "offloading program in bad state");
320 return -EINVAL;
321 }
322 return 0;
323}
324
325static bool
326nsim_map_key_match(struct bpf_map *map, struct nsim_map_entry *e, void *key)
327{
328 return e->key && !memcmp(key, e->key, map->key_size);
329}
330
331static int nsim_map_key_find(struct bpf_offloaded_map *offmap, void *key)
332{
333 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
334 unsigned int i;
335
336 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
337 if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
338 return i;
339
340 return -ENOENT;
341}
342
343static int
344nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
345{
346 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
347
348 nmap->entry[idx].key = kmalloc(offmap->map.key_size, GFP_USER);
349 if (!nmap->entry[idx].key)
350 return -ENOMEM;
351 nmap->entry[idx].value = kmalloc(offmap->map.value_size, GFP_USER);
352 if (!nmap->entry[idx].value) {
353 kfree(nmap->entry[idx].key);
354 nmap->entry[idx].key = NULL;
355 return -ENOMEM;
356 }
357
358 return 0;
359}
360
361static int
362nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
363 void *key, void *next_key)
364{
365 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
366 int idx = -ENOENT;
367
368 mutex_lock(&nmap->mutex);
369
370 if (key)
371 idx = nsim_map_key_find(offmap, key);
372 if (idx == -ENOENT)
373 idx = 0;
374 else
375 idx++;
376
377 for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
378 if (nmap->entry[idx].key) {
379 memcpy(next_key, nmap->entry[idx].key,
380 offmap->map.key_size);
381 break;
382 }
383 }
384
385 mutex_unlock(&nmap->mutex);
386
387 if (idx == ARRAY_SIZE(nmap->entry))
388 return -ENOENT;
389 return 0;
390}
391
392static int
393nsim_map_lookup_elem(struct bpf_offloaded_map *offmap, void *key, void *value)
394{
395 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
396 int idx;
397
398 mutex_lock(&nmap->mutex);
399
400 idx = nsim_map_key_find(offmap, key);
401 if (idx >= 0)
402 memcpy(value, nmap->entry[idx].value, offmap->map.value_size);
403
404 mutex_unlock(&nmap->mutex);
405
406 return idx < 0 ? idx : 0;
407}
408
409static int
410nsim_map_update_elem(struct bpf_offloaded_map *offmap,
411 void *key, void *value, u64 flags)
412{
413 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
414 int idx, err = 0;
415
416 mutex_lock(&nmap->mutex);
417
418 idx = nsim_map_key_find(offmap, key);
419 if (idx < 0 && flags == BPF_EXIST) {
420 err = idx;
421 goto exit_unlock;
422 }
423 if (idx >= 0 && flags == BPF_NOEXIST) {
424 err = -EEXIST;
425 goto exit_unlock;
426 }
427
428 if (idx < 0) {
429 for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
430 if (!nmap->entry[idx].key)
431 break;
432 if (idx == ARRAY_SIZE(nmap->entry)) {
433 err = -E2BIG;
434 goto exit_unlock;
435 }
436
437 err = nsim_map_alloc_elem(offmap, idx);
438 if (err)
439 goto exit_unlock;
440 }
441
442 memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
443 memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
444exit_unlock:
445 mutex_unlock(&nmap->mutex);
446
447 return err;
448}
449
450static int nsim_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
451{
452 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
453 int idx;
454
455 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
456 return -EINVAL;
457
458 mutex_lock(&nmap->mutex);
459
460 idx = nsim_map_key_find(offmap, key);
461 if (idx >= 0) {
462 kfree(nmap->entry[idx].key);
463 kfree(nmap->entry[idx].value);
464 memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
465 }
466
467 mutex_unlock(&nmap->mutex);
468
469 return idx < 0 ? idx : 0;
470}
471
472static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
473 .map_get_next_key = nsim_map_get_next_key,
474 .map_lookup_elem = nsim_map_lookup_elem,
475 .map_update_elem = nsim_map_update_elem,
476 .map_delete_elem = nsim_map_delete_elem,
477};
478
479static int
480nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
481{
482 struct nsim_bpf_bound_map *nmap;
483 int i, err;
484
485 if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
486 offmap->map.map_type != BPF_MAP_TYPE_HASH))
487 return -EINVAL;
488 if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
489 return -ENOMEM;
490 if (offmap->map.map_flags)
491 return -EINVAL;
492
493 nmap = kzalloc(sizeof(*nmap), GFP_USER);
494 if (!nmap)
495 return -ENOMEM;
496
497 offmap->dev_priv = nmap;
498 nmap->ns = ns;
499 nmap->map = offmap;
500 mutex_init(&nmap->mutex);
501
502 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
503 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
504 u32 *key;
505
506 err = nsim_map_alloc_elem(offmap, i);
507 if (err)
508 goto err_free;
509 key = nmap->entry[i].key;
510 *key = i;
511 }
512 }
513
514 offmap->dev_ops = &nsim_bpf_map_ops;
515 list_add_tail(&nmap->l, &ns->bpf_bound_maps);
516
517 return 0;
518
519err_free:
520 while (--i >= 0) {
521 kfree(nmap->entry[i].key);
522 kfree(nmap->entry[i].value);
523 }
524 kfree(nmap);
525 return err;
526}
527
528static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
529{
530 struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
531 unsigned int i;
532
533 for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
534 kfree(nmap->entry[i].key);
535 kfree(nmap->entry[i].value);
536 }
537 list_del_init(&nmap->l);
538 mutex_destroy(&nmap->mutex);
539 kfree(nmap);
540}
541
542int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
543{
544 struct netdevsim *ns = netdev_priv(dev);
545 struct nsim_bpf_bound_prog *state;
546 int err;
547
548 ASSERT_RTNL();
549
550 switch (bpf->command) {
551 case BPF_OFFLOAD_VERIFIER_PREP:
552 if (!ns->bpf_bind_accept)
553 return -EOPNOTSUPP;
554
555 err = nsim_bpf_create_prog(ns, bpf->verifier.prog);
556 if (err)
557 return err;
558
559 bpf->verifier.ops = &nsim_bpf_analyzer_ops;
560 return 0;
561 case BPF_OFFLOAD_TRANSLATE:
562 state = bpf->offload.prog->aux->offload->dev_priv;
563
564 state->state = "xlated";
565 return 0;
566 case BPF_OFFLOAD_DESTROY:
567 nsim_bpf_destroy_prog(bpf->offload.prog);
568 return 0;
569 case XDP_QUERY_PROG:
570 bpf->prog_attached = ns->xdp_prog_mode;
571 bpf->prog_id = ns->xdp_prog ? ns->xdp_prog->aux->id : 0;
572 bpf->prog_flags = ns->xdp_prog ? ns->xdp_flags : 0;
573 return 0;
574 case XDP_SETUP_PROG:
575 err = nsim_setup_prog_checks(ns, bpf);
576 if (err)
577 return err;
578
579 return nsim_xdp_set_prog(ns, bpf);
580 case XDP_SETUP_PROG_HW:
581 err = nsim_setup_prog_hw_checks(ns, bpf);
582 if (err)
583 return err;
584
585 return nsim_xdp_set_prog(ns, bpf);
586 case BPF_OFFLOAD_MAP_ALLOC:
587 if (!ns->bpf_map_accept)
588 return -EOPNOTSUPP;
589
590 return nsim_bpf_map_alloc(ns, bpf->offmap);
591 case BPF_OFFLOAD_MAP_FREE:
592 nsim_bpf_map_free(bpf->offmap);
593 return 0;
594 default:
595 return -EINVAL;
596 }
597}
598
599int nsim_bpf_init(struct netdevsim *ns)
600{
601 INIT_LIST_HEAD(&ns->bpf_bound_progs);
602 INIT_LIST_HEAD(&ns->bpf_bound_maps);
603
604 debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
605 &ns->bpf_offloaded_id);
606
607 ns->bpf_bind_accept = true;
608 debugfs_create_bool("bpf_bind_accept", 0600, ns->ddir,
609 &ns->bpf_bind_accept);
610 debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
611 &ns->bpf_bind_verifier_delay);
612 ns->ddir_bpf_bound_progs =
613 debugfs_create_dir("bpf_bound_progs", ns->ddir);
614 if (IS_ERR_OR_NULL(ns->ddir_bpf_bound_progs))
615 return -ENOMEM;
616
617 ns->bpf_tc_accept = true;
618 debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
619 &ns->bpf_tc_accept);
620 debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ns->ddir,
621 &ns->bpf_tc_non_bound_accept);
622 ns->bpf_xdpdrv_accept = true;
623 debugfs_create_bool("bpf_xdpdrv_accept", 0600, ns->ddir,
624 &ns->bpf_xdpdrv_accept);
625 ns->bpf_xdpoffload_accept = true;
626 debugfs_create_bool("bpf_xdpoffload_accept", 0600, ns->ddir,
627 &ns->bpf_xdpoffload_accept);
628
629 ns->bpf_map_accept = true;
630 debugfs_create_bool("bpf_map_accept", 0600, ns->ddir,
631 &ns->bpf_map_accept);
632
633 return 0;
634}
635
636void nsim_bpf_uninit(struct netdevsim *ns)
637{
638 WARN_ON(!list_empty(&ns->bpf_bound_progs));
639 WARN_ON(!list_empty(&ns->bpf_bound_maps));
640 WARN_ON(ns->xdp_prog);
641 WARN_ON(ns->bpf_offloaded);
642}