Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2023 Isovalent */
3
4#include <linux/netdevice.h>
5#include <linux/ethtool.h>
6#include <linux/etherdevice.h>
7#include <linux/filter.h>
8#include <linux/netfilter_netdev.h>
9#include <linux/bpf_mprog.h>
10#include <linux/indirect_call_wrapper.h>
11
12#include <net/netkit.h>
13#include <net/dst.h>
14#include <net/tcx.h>
15
16#define DRV_NAME "netkit"
17
18struct netkit {
19 /* Needed in fast-path */
20 struct net_device __rcu *peer;
21 struct bpf_mprog_entry __rcu *active;
22 enum netkit_action policy;
23 enum netkit_scrub scrub;
24 struct bpf_mprog_bundle bundle;
25
26 /* Needed in slow-path */
27 enum netkit_mode mode;
28 bool primary;
29 u32 headroom;
30};
31
32struct netkit_link {
33 struct bpf_link link;
34 struct net_device *dev;
35 u32 location;
36};
37
38static __always_inline int
39netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
40 enum netkit_action ret)
41{
42 const struct bpf_mprog_fp *fp;
43 const struct bpf_prog *prog;
44
45 bpf_mprog_foreach_prog(entry, fp, prog) {
46 bpf_compute_data_pointers(skb);
47 ret = bpf_prog_run(prog, skb);
48 if (ret != NETKIT_NEXT)
49 break;
50 }
51 return ret;
52}
53
54static void netkit_xnet(struct sk_buff *skb)
55{
56 skb->priority = 0;
57 skb->mark = 0;
58}
59
60static void netkit_prep_forward(struct sk_buff *skb,
61 bool xnet, bool xnet_scrub)
62{
63 skb_scrub_packet(skb, false);
64 nf_skip_egress(skb, true);
65 skb_reset_mac_header(skb);
66 if (!xnet)
67 return;
68 ipvs_reset(skb);
69 skb_clear_tstamp(skb);
70 if (xnet_scrub)
71 netkit_xnet(skb);
72}
73
74static struct netkit *netkit_priv(const struct net_device *dev)
75{
76 return netdev_priv(dev);
77}
78
79static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
80{
81 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
82 struct netkit *nk = netkit_priv(dev);
83 enum netkit_action ret = READ_ONCE(nk->policy);
84 netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
85 const struct bpf_mprog_entry *entry;
86 struct net_device *peer;
87 int len = skb->len;
88
89 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
90 rcu_read_lock();
91 peer = rcu_dereference(nk->peer);
92 if (unlikely(!peer || !(peer->flags & IFF_UP) ||
93 !pskb_may_pull(skb, ETH_HLEN) ||
94 skb_orphan_frags(skb, GFP_ATOMIC)))
95 goto drop;
96 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)),
97 nk->scrub);
98 eth_skb_pkt_type(skb, peer);
99 skb->dev = peer;
100 entry = rcu_dereference(nk->active);
101 if (entry)
102 ret = netkit_run(entry, skb, ret);
103 switch (ret) {
104 case NETKIT_NEXT:
105 case NETKIT_PASS:
106 eth_skb_pull_mac(skb);
107 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
108 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
109 dev_sw_netstats_tx_add(dev, 1, len);
110 dev_sw_netstats_rx_add(peer, len);
111 } else {
112 goto drop_stats;
113 }
114 break;
115 case NETKIT_REDIRECT:
116 dev_sw_netstats_tx_add(dev, 1, len);
117 skb_do_redirect(skb);
118 break;
119 case NETKIT_DROP:
120 default:
121drop:
122 kfree_skb(skb);
123drop_stats:
124 dev_core_stats_tx_dropped_inc(dev);
125 ret_dev = NET_XMIT_DROP;
126 break;
127 }
128 rcu_read_unlock();
129 bpf_net_ctx_clear(bpf_net_ctx);
130 return ret_dev;
131}
132
133static int netkit_open(struct net_device *dev)
134{
135 struct netkit *nk = netkit_priv(dev);
136 struct net_device *peer = rtnl_dereference(nk->peer);
137
138 if (!peer)
139 return -ENOTCONN;
140 if (peer->flags & IFF_UP) {
141 netif_carrier_on(dev);
142 netif_carrier_on(peer);
143 }
144 return 0;
145}
146
147static int netkit_close(struct net_device *dev)
148{
149 struct netkit *nk = netkit_priv(dev);
150 struct net_device *peer = rtnl_dereference(nk->peer);
151
152 netif_carrier_off(dev);
153 if (peer)
154 netif_carrier_off(peer);
155 return 0;
156}
157
158static int netkit_get_iflink(const struct net_device *dev)
159{
160 struct netkit *nk = netkit_priv(dev);
161 struct net_device *peer;
162 int iflink = 0;
163
164 rcu_read_lock();
165 peer = rcu_dereference(nk->peer);
166 if (peer)
167 iflink = READ_ONCE(peer->ifindex);
168 rcu_read_unlock();
169 return iflink;
170}
171
172static void netkit_set_multicast(struct net_device *dev)
173{
174 /* Nothing to do, we receive whatever gets pushed to us! */
175}
176
177static int netkit_set_macaddr(struct net_device *dev, void *sa)
178{
179 struct netkit *nk = netkit_priv(dev);
180
181 if (nk->mode != NETKIT_L2)
182 return -EOPNOTSUPP;
183
184 return eth_mac_addr(dev, sa);
185}
186
187static void netkit_set_headroom(struct net_device *dev, int headroom)
188{
189 struct netkit *nk = netkit_priv(dev), *nk2;
190 struct net_device *peer;
191
192 if (headroom < 0)
193 headroom = NET_SKB_PAD;
194
195 rcu_read_lock();
196 peer = rcu_dereference(nk->peer);
197 if (unlikely(!peer))
198 goto out;
199
200 nk2 = netkit_priv(peer);
201 nk->headroom = headroom;
202 headroom = max(nk->headroom, nk2->headroom);
203
204 peer->needed_headroom = headroom;
205 dev->needed_headroom = headroom;
206out:
207 rcu_read_unlock();
208}
209
210INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
211{
212 return rcu_dereference(netkit_priv(dev)->peer);
213}
214
215static void netkit_get_stats(struct net_device *dev,
216 struct rtnl_link_stats64 *stats)
217{
218 dev_fetch_sw_netstats(stats, dev->tstats);
219 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
220}
221
222static void netkit_uninit(struct net_device *dev);
223
224static const struct net_device_ops netkit_netdev_ops = {
225 .ndo_open = netkit_open,
226 .ndo_stop = netkit_close,
227 .ndo_start_xmit = netkit_xmit,
228 .ndo_set_rx_mode = netkit_set_multicast,
229 .ndo_set_rx_headroom = netkit_set_headroom,
230 .ndo_set_mac_address = netkit_set_macaddr,
231 .ndo_get_iflink = netkit_get_iflink,
232 .ndo_get_peer_dev = netkit_peer_dev,
233 .ndo_get_stats64 = netkit_get_stats,
234 .ndo_uninit = netkit_uninit,
235 .ndo_features_check = passthru_features_check,
236};
237
238static void netkit_get_drvinfo(struct net_device *dev,
239 struct ethtool_drvinfo *info)
240{
241 strscpy(info->driver, DRV_NAME, sizeof(info->driver));
242}
243
244static const struct ethtool_ops netkit_ethtool_ops = {
245 .get_drvinfo = netkit_get_drvinfo,
246};
247
248static void netkit_setup(struct net_device *dev)
249{
250 static const netdev_features_t netkit_features_hw_vlan =
251 NETIF_F_HW_VLAN_CTAG_TX |
252 NETIF_F_HW_VLAN_CTAG_RX |
253 NETIF_F_HW_VLAN_STAG_TX |
254 NETIF_F_HW_VLAN_STAG_RX;
255 static const netdev_features_t netkit_features =
256 netkit_features_hw_vlan |
257 NETIF_F_SG |
258 NETIF_F_FRAGLIST |
259 NETIF_F_HW_CSUM |
260 NETIF_F_RXCSUM |
261 NETIF_F_SCTP_CRC |
262 NETIF_F_HIGHDMA |
263 NETIF_F_GSO_SOFTWARE |
264 NETIF_F_GSO_ENCAP_ALL;
265
266 ether_setup(dev);
267 dev->max_mtu = ETH_MAX_MTU;
268 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
269
270 dev->flags |= IFF_NOARP;
271 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
272 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
273 dev->priv_flags |= IFF_PHONY_HEADROOM;
274 dev->priv_flags |= IFF_NO_QUEUE;
275 dev->priv_flags |= IFF_DISABLE_NETPOLL;
276 dev->lltx = true;
277
278 dev->ethtool_ops = &netkit_ethtool_ops;
279 dev->netdev_ops = &netkit_netdev_ops;
280
281 dev->features |= netkit_features;
282 dev->hw_features = netkit_features;
283 dev->hw_enc_features = netkit_features;
284 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
285 dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
286
287 dev->needs_free_netdev = true;
288
289 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
290}
291
292static struct net *netkit_get_link_net(const struct net_device *dev)
293{
294 struct netkit *nk = netkit_priv(dev);
295 struct net_device *peer = rtnl_dereference(nk->peer);
296
297 return peer ? dev_net(peer) : dev_net(dev);
298}
299
300static int netkit_check_policy(int policy, struct nlattr *tb,
301 struct netlink_ext_ack *extack)
302{
303 switch (policy) {
304 case NETKIT_PASS:
305 case NETKIT_DROP:
306 return 0;
307 default:
308 NL_SET_ERR_MSG_ATTR(extack, tb,
309 "Provided default xmit policy not supported");
310 return -EINVAL;
311 }
312}
313
314static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
315 struct netlink_ext_ack *extack)
316{
317 struct nlattr *attr = tb[IFLA_ADDRESS];
318
319 if (!attr)
320 return 0;
321 if (nla_len(attr) != ETH_ALEN)
322 return -EINVAL;
323 if (!is_valid_ether_addr(nla_data(attr)))
324 return -EADDRNOTAVAIL;
325 return 0;
326}
327
328static struct rtnl_link_ops netkit_link_ops;
329
330static int netkit_new_link(struct net *peer_net, struct net_device *dev,
331 struct nlattr *tb[], struct nlattr *data[],
332 struct netlink_ext_ack *extack)
333{
334 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr;
335 enum netkit_action policy_prim = NETKIT_PASS;
336 enum netkit_action policy_peer = NETKIT_PASS;
337 enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT;
338 enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT;
339 enum netkit_mode mode = NETKIT_L3;
340 unsigned char ifname_assign_type;
341 struct ifinfomsg *ifmp = NULL;
342 struct net_device *peer;
343 char ifname[IFNAMSIZ];
344 struct netkit *nk;
345 int err;
346
347 if (data) {
348 if (data[IFLA_NETKIT_MODE])
349 mode = nla_get_u32(data[IFLA_NETKIT_MODE]);
350 if (data[IFLA_NETKIT_PEER_INFO]) {
351 attr = data[IFLA_NETKIT_PEER_INFO];
352 ifmp = nla_data(attr);
353 rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
354 tbp = peer_tb;
355 }
356 if (data[IFLA_NETKIT_SCRUB])
357 scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]);
358 if (data[IFLA_NETKIT_PEER_SCRUB])
359 scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]);
360 if (data[IFLA_NETKIT_POLICY]) {
361 attr = data[IFLA_NETKIT_POLICY];
362 policy_prim = nla_get_u32(attr);
363 err = netkit_check_policy(policy_prim, attr, extack);
364 if (err < 0)
365 return err;
366 }
367 if (data[IFLA_NETKIT_PEER_POLICY]) {
368 attr = data[IFLA_NETKIT_PEER_POLICY];
369 policy_peer = nla_get_u32(attr);
370 err = netkit_check_policy(policy_peer, attr, extack);
371 if (err < 0)
372 return err;
373 }
374 }
375
376 if (ifmp && tbp[IFLA_IFNAME]) {
377 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
378 ifname_assign_type = NET_NAME_USER;
379 } else {
380 strscpy(ifname, "nk%d", IFNAMSIZ);
381 ifname_assign_type = NET_NAME_ENUM;
382 }
383 if (mode != NETKIT_L2 &&
384 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
385 return -EOPNOTSUPP;
386
387 peer = rtnl_create_link(peer_net, ifname, ifname_assign_type,
388 &netkit_link_ops, tbp, extack);
389 if (IS_ERR(peer))
390 return PTR_ERR(peer);
391
392 netif_inherit_tso_max(peer, dev);
393
394 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
395 eth_hw_addr_random(peer);
396 if (ifmp && dev->ifindex)
397 peer->ifindex = ifmp->ifi_index;
398
399 nk = netkit_priv(peer);
400 nk->primary = false;
401 nk->policy = policy_peer;
402 nk->scrub = scrub_peer;
403 nk->mode = mode;
404 bpf_mprog_bundle_init(&nk->bundle);
405
406 err = register_netdevice(peer);
407 if (err < 0)
408 goto err_register_peer;
409 netif_carrier_off(peer);
410 if (mode == NETKIT_L2)
411 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
412
413 err = rtnl_configure_link(peer, NULL, 0, NULL);
414 if (err < 0)
415 goto err_configure_peer;
416
417 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
418 eth_hw_addr_random(dev);
419 if (tb[IFLA_IFNAME])
420 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
421 else
422 strscpy(dev->name, "nk%d", IFNAMSIZ);
423
424 nk = netkit_priv(dev);
425 nk->primary = true;
426 nk->policy = policy_prim;
427 nk->scrub = scrub_prim;
428 nk->mode = mode;
429 bpf_mprog_bundle_init(&nk->bundle);
430
431 err = register_netdevice(dev);
432 if (err < 0)
433 goto err_configure_peer;
434 netif_carrier_off(dev);
435 if (mode == NETKIT_L2)
436 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
437
438 rcu_assign_pointer(netkit_priv(dev)->peer, peer);
439 rcu_assign_pointer(netkit_priv(peer)->peer, dev);
440 return 0;
441err_configure_peer:
442 unregister_netdevice(peer);
443 return err;
444err_register_peer:
445 free_netdev(peer);
446 return err;
447}
448
449static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
450 bool bundle_fallback)
451{
452 struct netkit *nk = netkit_priv(dev);
453 struct bpf_mprog_entry *entry;
454
455 ASSERT_RTNL();
456 entry = rcu_dereference_rtnl(nk->active);
457 if (entry)
458 return entry;
459 if (bundle_fallback)
460 return &nk->bundle.a;
461 return NULL;
462}
463
464static void netkit_entry_update(struct net_device *dev,
465 struct bpf_mprog_entry *entry)
466{
467 struct netkit *nk = netkit_priv(dev);
468
469 ASSERT_RTNL();
470 rcu_assign_pointer(nk->active, entry);
471}
472
473static void netkit_entry_sync(void)
474{
475 synchronize_rcu();
476}
477
478static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
479{
480 struct net_device *dev;
481 struct netkit *nk;
482
483 ASSERT_RTNL();
484
485 switch (which) {
486 case BPF_NETKIT_PRIMARY:
487 case BPF_NETKIT_PEER:
488 break;
489 default:
490 return ERR_PTR(-EINVAL);
491 }
492
493 dev = __dev_get_by_index(net, ifindex);
494 if (!dev)
495 return ERR_PTR(-ENODEV);
496 if (dev->netdev_ops != &netkit_netdev_ops)
497 return ERR_PTR(-ENXIO);
498
499 nk = netkit_priv(dev);
500 if (!nk->primary)
501 return ERR_PTR(-EACCES);
502 if (which == BPF_NETKIT_PEER) {
503 dev = rcu_dereference_rtnl(nk->peer);
504 if (!dev)
505 return ERR_PTR(-ENODEV);
506 }
507 return dev;
508}
509
510int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
511{
512 struct bpf_mprog_entry *entry, *entry_new;
513 struct bpf_prog *replace_prog = NULL;
514 struct net_device *dev;
515 int ret;
516
517 rtnl_lock();
518 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
519 attr->attach_type);
520 if (IS_ERR(dev)) {
521 ret = PTR_ERR(dev);
522 goto out;
523 }
524 entry = netkit_entry_fetch(dev, true);
525 if (attr->attach_flags & BPF_F_REPLACE) {
526 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
527 prog->type);
528 if (IS_ERR(replace_prog)) {
529 ret = PTR_ERR(replace_prog);
530 replace_prog = NULL;
531 goto out;
532 }
533 }
534 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
535 attr->attach_flags, attr->relative_fd,
536 attr->expected_revision);
537 if (!ret) {
538 if (entry != entry_new) {
539 netkit_entry_update(dev, entry_new);
540 netkit_entry_sync();
541 }
542 bpf_mprog_commit(entry);
543 }
544out:
545 if (replace_prog)
546 bpf_prog_put(replace_prog);
547 rtnl_unlock();
548 return ret;
549}
550
551int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
552{
553 struct bpf_mprog_entry *entry, *entry_new;
554 struct net_device *dev;
555 int ret;
556
557 rtnl_lock();
558 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
559 attr->attach_type);
560 if (IS_ERR(dev)) {
561 ret = PTR_ERR(dev);
562 goto out;
563 }
564 entry = netkit_entry_fetch(dev, false);
565 if (!entry) {
566 ret = -ENOENT;
567 goto out;
568 }
569 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
570 attr->relative_fd, attr->expected_revision);
571 if (!ret) {
572 if (!bpf_mprog_total(entry_new))
573 entry_new = NULL;
574 netkit_entry_update(dev, entry_new);
575 netkit_entry_sync();
576 bpf_mprog_commit(entry);
577 }
578out:
579 rtnl_unlock();
580 return ret;
581}
582
583int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
584{
585 struct net_device *dev;
586 int ret;
587
588 rtnl_lock();
589 dev = netkit_dev_fetch(current->nsproxy->net_ns,
590 attr->query.target_ifindex,
591 attr->query.attach_type);
592 if (IS_ERR(dev)) {
593 ret = PTR_ERR(dev);
594 goto out;
595 }
596 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
597out:
598 rtnl_unlock();
599 return ret;
600}
601
602static struct netkit_link *netkit_link(const struct bpf_link *link)
603{
604 return container_of(link, struct netkit_link, link);
605}
606
607static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
608 u32 id_or_fd, u64 revision)
609{
610 struct netkit_link *nkl = netkit_link(link);
611 struct bpf_mprog_entry *entry, *entry_new;
612 struct net_device *dev = nkl->dev;
613 int ret;
614
615 ASSERT_RTNL();
616 entry = netkit_entry_fetch(dev, true);
617 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
618 id_or_fd, revision);
619 if (!ret) {
620 if (entry != entry_new) {
621 netkit_entry_update(dev, entry_new);
622 netkit_entry_sync();
623 }
624 bpf_mprog_commit(entry);
625 }
626 return ret;
627}
628
629static void netkit_link_release(struct bpf_link *link)
630{
631 struct netkit_link *nkl = netkit_link(link);
632 struct bpf_mprog_entry *entry, *entry_new;
633 struct net_device *dev;
634 int ret = 0;
635
636 rtnl_lock();
637 dev = nkl->dev;
638 if (!dev)
639 goto out;
640 entry = netkit_entry_fetch(dev, false);
641 if (!entry) {
642 ret = -ENOENT;
643 goto out;
644 }
645 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
646 if (!ret) {
647 if (!bpf_mprog_total(entry_new))
648 entry_new = NULL;
649 netkit_entry_update(dev, entry_new);
650 netkit_entry_sync();
651 bpf_mprog_commit(entry);
652 nkl->dev = NULL;
653 }
654out:
655 WARN_ON_ONCE(ret);
656 rtnl_unlock();
657}
658
659static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
660 struct bpf_prog *oprog)
661{
662 struct netkit_link *nkl = netkit_link(link);
663 struct bpf_mprog_entry *entry, *entry_new;
664 struct net_device *dev;
665 int ret = 0;
666
667 rtnl_lock();
668 dev = nkl->dev;
669 if (!dev) {
670 ret = -ENOLINK;
671 goto out;
672 }
673 if (oprog && link->prog != oprog) {
674 ret = -EPERM;
675 goto out;
676 }
677 oprog = link->prog;
678 if (oprog == nprog) {
679 bpf_prog_put(nprog);
680 goto out;
681 }
682 entry = netkit_entry_fetch(dev, false);
683 if (!entry) {
684 ret = -ENOENT;
685 goto out;
686 }
687 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
688 BPF_F_REPLACE | BPF_F_ID,
689 link->prog->aux->id, 0);
690 if (!ret) {
691 WARN_ON_ONCE(entry != entry_new);
692 oprog = xchg(&link->prog, nprog);
693 bpf_prog_put(oprog);
694 bpf_mprog_commit(entry);
695 }
696out:
697 rtnl_unlock();
698 return ret;
699}
700
701static void netkit_link_dealloc(struct bpf_link *link)
702{
703 kfree(netkit_link(link));
704}
705
706static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
707{
708 const struct netkit_link *nkl = netkit_link(link);
709 u32 ifindex = 0;
710
711 rtnl_lock();
712 if (nkl->dev)
713 ifindex = nkl->dev->ifindex;
714 rtnl_unlock();
715
716 seq_printf(seq, "ifindex:\t%u\n", ifindex);
717 seq_printf(seq, "attach_type:\t%u (%s)\n",
718 nkl->location,
719 nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
720}
721
722static int netkit_link_fill_info(const struct bpf_link *link,
723 struct bpf_link_info *info)
724{
725 const struct netkit_link *nkl = netkit_link(link);
726 u32 ifindex = 0;
727
728 rtnl_lock();
729 if (nkl->dev)
730 ifindex = nkl->dev->ifindex;
731 rtnl_unlock();
732
733 info->netkit.ifindex = ifindex;
734 info->netkit.attach_type = nkl->location;
735 return 0;
736}
737
738static int netkit_link_detach(struct bpf_link *link)
739{
740 netkit_link_release(link);
741 return 0;
742}
743
744static const struct bpf_link_ops netkit_link_lops = {
745 .release = netkit_link_release,
746 .detach = netkit_link_detach,
747 .dealloc = netkit_link_dealloc,
748 .update_prog = netkit_link_update,
749 .show_fdinfo = netkit_link_fdinfo,
750 .fill_link_info = netkit_link_fill_info,
751};
752
753static int netkit_link_init(struct netkit_link *nkl,
754 struct bpf_link_primer *link_primer,
755 const union bpf_attr *attr,
756 struct net_device *dev,
757 struct bpf_prog *prog)
758{
759 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
760 &netkit_link_lops, prog);
761 nkl->location = attr->link_create.attach_type;
762 nkl->dev = dev;
763 return bpf_link_prime(&nkl->link, link_primer);
764}
765
766int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
767{
768 struct bpf_link_primer link_primer;
769 struct netkit_link *nkl;
770 struct net_device *dev;
771 int ret;
772
773 rtnl_lock();
774 dev = netkit_dev_fetch(current->nsproxy->net_ns,
775 attr->link_create.target_ifindex,
776 attr->link_create.attach_type);
777 if (IS_ERR(dev)) {
778 ret = PTR_ERR(dev);
779 goto out;
780 }
781 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
782 if (!nkl) {
783 ret = -ENOMEM;
784 goto out;
785 }
786 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
787 if (ret) {
788 kfree(nkl);
789 goto out;
790 }
791 ret = netkit_link_prog_attach(&nkl->link,
792 attr->link_create.flags,
793 attr->link_create.netkit.relative_fd,
794 attr->link_create.netkit.expected_revision);
795 if (ret) {
796 nkl->dev = NULL;
797 bpf_link_cleanup(&link_primer);
798 goto out;
799 }
800 ret = bpf_link_settle(&link_primer);
801out:
802 rtnl_unlock();
803 return ret;
804}
805
806static void netkit_release_all(struct net_device *dev)
807{
808 struct bpf_mprog_entry *entry;
809 struct bpf_tuple tuple = {};
810 struct bpf_mprog_fp *fp;
811 struct bpf_mprog_cp *cp;
812
813 entry = netkit_entry_fetch(dev, false);
814 if (!entry)
815 return;
816 netkit_entry_update(dev, NULL);
817 netkit_entry_sync();
818 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
819 if (tuple.link)
820 netkit_link(tuple.link)->dev = NULL;
821 else
822 bpf_prog_put(tuple.prog);
823 }
824}
825
826static void netkit_uninit(struct net_device *dev)
827{
828 netkit_release_all(dev);
829}
830
831static void netkit_del_link(struct net_device *dev, struct list_head *head)
832{
833 struct netkit *nk = netkit_priv(dev);
834 struct net_device *peer = rtnl_dereference(nk->peer);
835
836 RCU_INIT_POINTER(nk->peer, NULL);
837 unregister_netdevice_queue(dev, head);
838 if (peer) {
839 nk = netkit_priv(peer);
840 RCU_INIT_POINTER(nk->peer, NULL);
841 unregister_netdevice_queue(peer, head);
842 }
843}
844
845static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
846 struct nlattr *data[],
847 struct netlink_ext_ack *extack)
848{
849 struct netkit *nk = netkit_priv(dev);
850 struct net_device *peer = rtnl_dereference(nk->peer);
851 enum netkit_action policy;
852 struct nlattr *attr;
853 int err;
854
855 if (!nk->primary) {
856 NL_SET_ERR_MSG(extack,
857 "netkit link settings can be changed only through the primary device");
858 return -EACCES;
859 }
860
861 if (data[IFLA_NETKIT_MODE]) {
862 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE],
863 "netkit link operating mode cannot be changed after device creation");
864 return -EACCES;
865 }
866
867 if (data[IFLA_NETKIT_SCRUB]) {
868 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_SCRUB],
869 "netkit scrubbing cannot be changed after device creation");
870 return -EACCES;
871 }
872
873 if (data[IFLA_NETKIT_PEER_SCRUB]) {
874 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_SCRUB],
875 "netkit scrubbing cannot be changed after device creation");
876 return -EACCES;
877 }
878
879 if (data[IFLA_NETKIT_PEER_INFO]) {
880 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO],
881 "netkit peer info cannot be changed after device creation");
882 return -EINVAL;
883 }
884
885 if (data[IFLA_NETKIT_POLICY]) {
886 attr = data[IFLA_NETKIT_POLICY];
887 policy = nla_get_u32(attr);
888 err = netkit_check_policy(policy, attr, extack);
889 if (err)
890 return err;
891 WRITE_ONCE(nk->policy, policy);
892 }
893
894 if (data[IFLA_NETKIT_PEER_POLICY]) {
895 err = -EOPNOTSUPP;
896 attr = data[IFLA_NETKIT_PEER_POLICY];
897 policy = nla_get_u32(attr);
898 if (peer)
899 err = netkit_check_policy(policy, attr, extack);
900 if (err)
901 return err;
902 nk = netkit_priv(peer);
903 WRITE_ONCE(nk->policy, policy);
904 }
905
906 return 0;
907}
908
909static size_t netkit_get_size(const struct net_device *dev)
910{
911 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
912 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
913 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */
914 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */
915 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
916 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */
917 0;
918}
919
920static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
921{
922 struct netkit *nk = netkit_priv(dev);
923 struct net_device *peer = rtnl_dereference(nk->peer);
924
925 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
926 return -EMSGSIZE;
927 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
928 return -EMSGSIZE;
929 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
930 return -EMSGSIZE;
931 if (nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub))
932 return -EMSGSIZE;
933
934 if (peer) {
935 nk = netkit_priv(peer);
936 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
937 return -EMSGSIZE;
938 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub))
939 return -EMSGSIZE;
940 }
941
942 return 0;
943}
944
945static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
946 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) },
947 [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3),
948 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 },
949 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 },
950 [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
951 [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT),
952 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT,
953 .reject_message = "Primary attribute is read-only" },
954};
955
956static struct rtnl_link_ops netkit_link_ops = {
957 .kind = DRV_NAME,
958 .priv_size = sizeof(struct netkit),
959 .setup = netkit_setup,
960 .newlink = netkit_new_link,
961 .dellink = netkit_del_link,
962 .changelink = netkit_change_link,
963 .get_link_net = netkit_get_link_net,
964 .get_size = netkit_get_size,
965 .fill_info = netkit_fill_info,
966 .policy = netkit_policy,
967 .validate = netkit_validate,
968 .peer_type = IFLA_NETKIT_PEER_INFO,
969 .maxtype = IFLA_NETKIT_MAX,
970};
971
972static __init int netkit_init(void)
973{
974 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
975 (int)NETKIT_PASS != (int)TCX_PASS ||
976 (int)NETKIT_DROP != (int)TCX_DROP ||
977 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
978
979 return rtnl_link_register(&netkit_link_ops);
980}
981
982static __exit void netkit_exit(void)
983{
984 rtnl_link_unregister(&netkit_link_ops);
985}
986
987module_init(netkit_init);
988module_exit(netkit_exit);
989
990MODULE_DESCRIPTION("BPF-programmable network device");
991MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");
992MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");
993MODULE_LICENSE("GPL");
994MODULE_ALIAS_RTNL_LINK(DRV_NAME);
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2023 Isovalent */
3
4#include <linux/netdevice.h>
5#include <linux/ethtool.h>
6#include <linux/etherdevice.h>
7#include <linux/filter.h>
8#include <linux/netfilter_netdev.h>
9#include <linux/bpf_mprog.h>
10#include <linux/indirect_call_wrapper.h>
11
12#include <net/netkit.h>
13#include <net/dst.h>
14#include <net/tcx.h>
15
16#define DRV_NAME "netkit"
17
18struct netkit {
19 /* Needed in fast-path */
20 struct net_device __rcu *peer;
21 struct bpf_mprog_entry __rcu *active;
22 enum netkit_action policy;
23 struct bpf_mprog_bundle bundle;
24
25 /* Needed in slow-path */
26 enum netkit_mode mode;
27 bool primary;
28 u32 headroom;
29};
30
31struct netkit_link {
32 struct bpf_link link;
33 struct net_device *dev;
34 u32 location;
35};
36
37static __always_inline int
38netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
39 enum netkit_action ret)
40{
41 const struct bpf_mprog_fp *fp;
42 const struct bpf_prog *prog;
43
44 bpf_mprog_foreach_prog(entry, fp, prog) {
45 bpf_compute_data_pointers(skb);
46 ret = bpf_prog_run(prog, skb);
47 if (ret != NETKIT_NEXT)
48 break;
49 }
50 return ret;
51}
52
53static void netkit_prep_forward(struct sk_buff *skb, bool xnet)
54{
55 skb_scrub_packet(skb, xnet);
56 skb->priority = 0;
57 nf_skip_egress(skb, true);
58 skb_reset_mac_header(skb);
59}
60
61static struct netkit *netkit_priv(const struct net_device *dev)
62{
63 return netdev_priv(dev);
64}
65
66static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
67{
68 struct netkit *nk = netkit_priv(dev);
69 enum netkit_action ret = READ_ONCE(nk->policy);
70 netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
71 const struct bpf_mprog_entry *entry;
72 struct net_device *peer;
73 int len = skb->len;
74
75 rcu_read_lock();
76 peer = rcu_dereference(nk->peer);
77 if (unlikely(!peer || !(peer->flags & IFF_UP) ||
78 !pskb_may_pull(skb, ETH_HLEN) ||
79 skb_orphan_frags(skb, GFP_ATOMIC)))
80 goto drop;
81 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)));
82 eth_skb_pkt_type(skb, peer);
83 skb->dev = peer;
84 entry = rcu_dereference(nk->active);
85 if (entry)
86 ret = netkit_run(entry, skb, ret);
87 switch (ret) {
88 case NETKIT_NEXT:
89 case NETKIT_PASS:
90 eth_skb_pull_mac(skb);
91 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
92 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
93 dev_sw_netstats_tx_add(dev, 1, len);
94 dev_sw_netstats_rx_add(peer, len);
95 } else {
96 goto drop_stats;
97 }
98 break;
99 case NETKIT_REDIRECT:
100 dev_sw_netstats_tx_add(dev, 1, len);
101 skb_do_redirect(skb);
102 break;
103 case NETKIT_DROP:
104 default:
105drop:
106 kfree_skb(skb);
107drop_stats:
108 dev_core_stats_tx_dropped_inc(dev);
109 ret_dev = NET_XMIT_DROP;
110 break;
111 }
112 rcu_read_unlock();
113 return ret_dev;
114}
115
116static int netkit_open(struct net_device *dev)
117{
118 struct netkit *nk = netkit_priv(dev);
119 struct net_device *peer = rtnl_dereference(nk->peer);
120
121 if (!peer)
122 return -ENOTCONN;
123 if (peer->flags & IFF_UP) {
124 netif_carrier_on(dev);
125 netif_carrier_on(peer);
126 }
127 return 0;
128}
129
130static int netkit_close(struct net_device *dev)
131{
132 struct netkit *nk = netkit_priv(dev);
133 struct net_device *peer = rtnl_dereference(nk->peer);
134
135 netif_carrier_off(dev);
136 if (peer)
137 netif_carrier_off(peer);
138 return 0;
139}
140
141static int netkit_get_iflink(const struct net_device *dev)
142{
143 struct netkit *nk = netkit_priv(dev);
144 struct net_device *peer;
145 int iflink = 0;
146
147 rcu_read_lock();
148 peer = rcu_dereference(nk->peer);
149 if (peer)
150 iflink = READ_ONCE(peer->ifindex);
151 rcu_read_unlock();
152 return iflink;
153}
154
155static void netkit_set_multicast(struct net_device *dev)
156{
157 /* Nothing to do, we receive whatever gets pushed to us! */
158}
159
160static int netkit_set_macaddr(struct net_device *dev, void *sa)
161{
162 struct netkit *nk = netkit_priv(dev);
163
164 if (nk->mode != NETKIT_L2)
165 return -EOPNOTSUPP;
166
167 return eth_mac_addr(dev, sa);
168}
169
170static void netkit_set_headroom(struct net_device *dev, int headroom)
171{
172 struct netkit *nk = netkit_priv(dev), *nk2;
173 struct net_device *peer;
174
175 if (headroom < 0)
176 headroom = NET_SKB_PAD;
177
178 rcu_read_lock();
179 peer = rcu_dereference(nk->peer);
180 if (unlikely(!peer))
181 goto out;
182
183 nk2 = netkit_priv(peer);
184 nk->headroom = headroom;
185 headroom = max(nk->headroom, nk2->headroom);
186
187 peer->needed_headroom = headroom;
188 dev->needed_headroom = headroom;
189out:
190 rcu_read_unlock();
191}
192
193INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
194{
195 return rcu_dereference(netkit_priv(dev)->peer);
196}
197
198static void netkit_get_stats(struct net_device *dev,
199 struct rtnl_link_stats64 *stats)
200{
201 dev_fetch_sw_netstats(stats, dev->tstats);
202 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
203}
204
205static void netkit_uninit(struct net_device *dev);
206
207static const struct net_device_ops netkit_netdev_ops = {
208 .ndo_open = netkit_open,
209 .ndo_stop = netkit_close,
210 .ndo_start_xmit = netkit_xmit,
211 .ndo_set_rx_mode = netkit_set_multicast,
212 .ndo_set_rx_headroom = netkit_set_headroom,
213 .ndo_set_mac_address = netkit_set_macaddr,
214 .ndo_get_iflink = netkit_get_iflink,
215 .ndo_get_peer_dev = netkit_peer_dev,
216 .ndo_get_stats64 = netkit_get_stats,
217 .ndo_uninit = netkit_uninit,
218 .ndo_features_check = passthru_features_check,
219};
220
221static void netkit_get_drvinfo(struct net_device *dev,
222 struct ethtool_drvinfo *info)
223{
224 strscpy(info->driver, DRV_NAME, sizeof(info->driver));
225}
226
227static const struct ethtool_ops netkit_ethtool_ops = {
228 .get_drvinfo = netkit_get_drvinfo,
229};
230
231static void netkit_setup(struct net_device *dev)
232{
233 static const netdev_features_t netkit_features_hw_vlan =
234 NETIF_F_HW_VLAN_CTAG_TX |
235 NETIF_F_HW_VLAN_CTAG_RX |
236 NETIF_F_HW_VLAN_STAG_TX |
237 NETIF_F_HW_VLAN_STAG_RX;
238 static const netdev_features_t netkit_features =
239 netkit_features_hw_vlan |
240 NETIF_F_SG |
241 NETIF_F_FRAGLIST |
242 NETIF_F_HW_CSUM |
243 NETIF_F_RXCSUM |
244 NETIF_F_SCTP_CRC |
245 NETIF_F_HIGHDMA |
246 NETIF_F_GSO_SOFTWARE |
247 NETIF_F_GSO_ENCAP_ALL;
248
249 ether_setup(dev);
250 dev->max_mtu = ETH_MAX_MTU;
251 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
252
253 dev->flags |= IFF_NOARP;
254 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
255 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
256 dev->priv_flags |= IFF_PHONY_HEADROOM;
257 dev->priv_flags |= IFF_NO_QUEUE;
258
259 dev->ethtool_ops = &netkit_ethtool_ops;
260 dev->netdev_ops = &netkit_netdev_ops;
261
262 dev->features |= netkit_features | NETIF_F_LLTX;
263 dev->hw_features = netkit_features;
264 dev->hw_enc_features = netkit_features;
265 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
266 dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
267
268 dev->needs_free_netdev = true;
269
270 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
271}
272
273static struct net *netkit_get_link_net(const struct net_device *dev)
274{
275 struct netkit *nk = netkit_priv(dev);
276 struct net_device *peer = rtnl_dereference(nk->peer);
277
278 return peer ? dev_net(peer) : dev_net(dev);
279}
280
281static int netkit_check_policy(int policy, struct nlattr *tb,
282 struct netlink_ext_ack *extack)
283{
284 switch (policy) {
285 case NETKIT_PASS:
286 case NETKIT_DROP:
287 return 0;
288 default:
289 NL_SET_ERR_MSG_ATTR(extack, tb,
290 "Provided default xmit policy not supported");
291 return -EINVAL;
292 }
293}
294
295static int netkit_check_mode(int mode, struct nlattr *tb,
296 struct netlink_ext_ack *extack)
297{
298 switch (mode) {
299 case NETKIT_L2:
300 case NETKIT_L3:
301 return 0;
302 default:
303 NL_SET_ERR_MSG_ATTR(extack, tb,
304 "Provided device mode can only be L2 or L3");
305 return -EINVAL;
306 }
307}
308
309static int netkit_validate(struct nlattr *tb[], struct nlattr *data[],
310 struct netlink_ext_ack *extack)
311{
312 struct nlattr *attr = tb[IFLA_ADDRESS];
313
314 if (!attr)
315 return 0;
316 if (nla_len(attr) != ETH_ALEN)
317 return -EINVAL;
318 if (!is_valid_ether_addr(nla_data(attr)))
319 return -EADDRNOTAVAIL;
320 return 0;
321}
322
323static struct rtnl_link_ops netkit_link_ops;
324
325static int netkit_new_link(struct net *src_net, struct net_device *dev,
326 struct nlattr *tb[], struct nlattr *data[],
327 struct netlink_ext_ack *extack)
328{
329 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr;
330 enum netkit_action default_prim = NETKIT_PASS;
331 enum netkit_action default_peer = NETKIT_PASS;
332 enum netkit_mode mode = NETKIT_L3;
333 unsigned char ifname_assign_type;
334 struct ifinfomsg *ifmp = NULL;
335 struct net_device *peer;
336 char ifname[IFNAMSIZ];
337 struct netkit *nk;
338 struct net *net;
339 int err;
340
341 if (data) {
342 if (data[IFLA_NETKIT_MODE]) {
343 attr = data[IFLA_NETKIT_MODE];
344 mode = nla_get_u32(attr);
345 err = netkit_check_mode(mode, attr, extack);
346 if (err < 0)
347 return err;
348 }
349 if (data[IFLA_NETKIT_PEER_INFO]) {
350 attr = data[IFLA_NETKIT_PEER_INFO];
351 ifmp = nla_data(attr);
352 err = rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack);
353 if (err < 0)
354 return err;
355 err = netkit_validate(peer_tb, NULL, extack);
356 if (err < 0)
357 return err;
358 tbp = peer_tb;
359 }
360 if (data[IFLA_NETKIT_POLICY]) {
361 attr = data[IFLA_NETKIT_POLICY];
362 default_prim = nla_get_u32(attr);
363 err = netkit_check_policy(default_prim, attr, extack);
364 if (err < 0)
365 return err;
366 }
367 if (data[IFLA_NETKIT_PEER_POLICY]) {
368 attr = data[IFLA_NETKIT_PEER_POLICY];
369 default_peer = nla_get_u32(attr);
370 err = netkit_check_policy(default_peer, attr, extack);
371 if (err < 0)
372 return err;
373 }
374 }
375
376 if (ifmp && tbp[IFLA_IFNAME]) {
377 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
378 ifname_assign_type = NET_NAME_USER;
379 } else {
380 strscpy(ifname, "nk%d", IFNAMSIZ);
381 ifname_assign_type = NET_NAME_ENUM;
382 }
383 if (mode != NETKIT_L2 &&
384 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS]))
385 return -EOPNOTSUPP;
386
387 net = rtnl_link_get_net(src_net, tbp);
388 if (IS_ERR(net))
389 return PTR_ERR(net);
390
391 peer = rtnl_create_link(net, ifname, ifname_assign_type,
392 &netkit_link_ops, tbp, extack);
393 if (IS_ERR(peer)) {
394 put_net(net);
395 return PTR_ERR(peer);
396 }
397
398 netif_inherit_tso_max(peer, dev);
399
400 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS]))
401 eth_hw_addr_random(peer);
402 if (ifmp && dev->ifindex)
403 peer->ifindex = ifmp->ifi_index;
404
405 nk = netkit_priv(peer);
406 nk->primary = false;
407 nk->policy = default_peer;
408 nk->mode = mode;
409 bpf_mprog_bundle_init(&nk->bundle);
410
411 err = register_netdevice(peer);
412 put_net(net);
413 if (err < 0)
414 goto err_register_peer;
415 netif_carrier_off(peer);
416 if (mode == NETKIT_L2)
417 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
418
419 err = rtnl_configure_link(peer, NULL, 0, NULL);
420 if (err < 0)
421 goto err_configure_peer;
422
423 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS])
424 eth_hw_addr_random(dev);
425 if (tb[IFLA_IFNAME])
426 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
427 else
428 strscpy(dev->name, "nk%d", IFNAMSIZ);
429
430 nk = netkit_priv(dev);
431 nk->primary = true;
432 nk->policy = default_prim;
433 nk->mode = mode;
434 bpf_mprog_bundle_init(&nk->bundle);
435
436 err = register_netdevice(dev);
437 if (err < 0)
438 goto err_configure_peer;
439 netif_carrier_off(dev);
440 if (mode == NETKIT_L2)
441 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
442
443 rcu_assign_pointer(netkit_priv(dev)->peer, peer);
444 rcu_assign_pointer(netkit_priv(peer)->peer, dev);
445 return 0;
446err_configure_peer:
447 unregister_netdevice(peer);
448 return err;
449err_register_peer:
450 free_netdev(peer);
451 return err;
452}
453
454static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev,
455 bool bundle_fallback)
456{
457 struct netkit *nk = netkit_priv(dev);
458 struct bpf_mprog_entry *entry;
459
460 ASSERT_RTNL();
461 entry = rcu_dereference_rtnl(nk->active);
462 if (entry)
463 return entry;
464 if (bundle_fallback)
465 return &nk->bundle.a;
466 return NULL;
467}
468
469static void netkit_entry_update(struct net_device *dev,
470 struct bpf_mprog_entry *entry)
471{
472 struct netkit *nk = netkit_priv(dev);
473
474 ASSERT_RTNL();
475 rcu_assign_pointer(nk->active, entry);
476}
477
478static void netkit_entry_sync(void)
479{
480 synchronize_rcu();
481}
482
483static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which)
484{
485 struct net_device *dev;
486 struct netkit *nk;
487
488 ASSERT_RTNL();
489
490 switch (which) {
491 case BPF_NETKIT_PRIMARY:
492 case BPF_NETKIT_PEER:
493 break;
494 default:
495 return ERR_PTR(-EINVAL);
496 }
497
498 dev = __dev_get_by_index(net, ifindex);
499 if (!dev)
500 return ERR_PTR(-ENODEV);
501 if (dev->netdev_ops != &netkit_netdev_ops)
502 return ERR_PTR(-ENXIO);
503
504 nk = netkit_priv(dev);
505 if (!nk->primary)
506 return ERR_PTR(-EACCES);
507 if (which == BPF_NETKIT_PEER) {
508 dev = rcu_dereference_rtnl(nk->peer);
509 if (!dev)
510 return ERR_PTR(-ENODEV);
511 }
512 return dev;
513}
514
515int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
516{
517 struct bpf_mprog_entry *entry, *entry_new;
518 struct bpf_prog *replace_prog = NULL;
519 struct net_device *dev;
520 int ret;
521
522 rtnl_lock();
523 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
524 attr->attach_type);
525 if (IS_ERR(dev)) {
526 ret = PTR_ERR(dev);
527 goto out;
528 }
529 entry = netkit_entry_fetch(dev, true);
530 if (attr->attach_flags & BPF_F_REPLACE) {
531 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
532 prog->type);
533 if (IS_ERR(replace_prog)) {
534 ret = PTR_ERR(replace_prog);
535 replace_prog = NULL;
536 goto out;
537 }
538 }
539 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
540 attr->attach_flags, attr->relative_fd,
541 attr->expected_revision);
542 if (!ret) {
543 if (entry != entry_new) {
544 netkit_entry_update(dev, entry_new);
545 netkit_entry_sync();
546 }
547 bpf_mprog_commit(entry);
548 }
549out:
550 if (replace_prog)
551 bpf_prog_put(replace_prog);
552 rtnl_unlock();
553 return ret;
554}
555
556int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
557{
558 struct bpf_mprog_entry *entry, *entry_new;
559 struct net_device *dev;
560 int ret;
561
562 rtnl_lock();
563 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex,
564 attr->attach_type);
565 if (IS_ERR(dev)) {
566 ret = PTR_ERR(dev);
567 goto out;
568 }
569 entry = netkit_entry_fetch(dev, false);
570 if (!entry) {
571 ret = -ENOENT;
572 goto out;
573 }
574 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
575 attr->relative_fd, attr->expected_revision);
576 if (!ret) {
577 if (!bpf_mprog_total(entry_new))
578 entry_new = NULL;
579 netkit_entry_update(dev, entry_new);
580 netkit_entry_sync();
581 bpf_mprog_commit(entry);
582 }
583out:
584 rtnl_unlock();
585 return ret;
586}
587
588int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
589{
590 struct net_device *dev;
591 int ret;
592
593 rtnl_lock();
594 dev = netkit_dev_fetch(current->nsproxy->net_ns,
595 attr->query.target_ifindex,
596 attr->query.attach_type);
597 if (IS_ERR(dev)) {
598 ret = PTR_ERR(dev);
599 goto out;
600 }
601 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));
602out:
603 rtnl_unlock();
604 return ret;
605}
606
607static struct netkit_link *netkit_link(const struct bpf_link *link)
608{
609 return container_of(link, struct netkit_link, link);
610}
611
612static int netkit_link_prog_attach(struct bpf_link *link, u32 flags,
613 u32 id_or_fd, u64 revision)
614{
615 struct netkit_link *nkl = netkit_link(link);
616 struct bpf_mprog_entry *entry, *entry_new;
617 struct net_device *dev = nkl->dev;
618 int ret;
619
620 ASSERT_RTNL();
621 entry = netkit_entry_fetch(dev, true);
622 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
623 id_or_fd, revision);
624 if (!ret) {
625 if (entry != entry_new) {
626 netkit_entry_update(dev, entry_new);
627 netkit_entry_sync();
628 }
629 bpf_mprog_commit(entry);
630 }
631 return ret;
632}
633
634static void netkit_link_release(struct bpf_link *link)
635{
636 struct netkit_link *nkl = netkit_link(link);
637 struct bpf_mprog_entry *entry, *entry_new;
638 struct net_device *dev;
639 int ret = 0;
640
641 rtnl_lock();
642 dev = nkl->dev;
643 if (!dev)
644 goto out;
645 entry = netkit_entry_fetch(dev, false);
646 if (!entry) {
647 ret = -ENOENT;
648 goto out;
649 }
650 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
651 if (!ret) {
652 if (!bpf_mprog_total(entry_new))
653 entry_new = NULL;
654 netkit_entry_update(dev, entry_new);
655 netkit_entry_sync();
656 bpf_mprog_commit(entry);
657 nkl->dev = NULL;
658 }
659out:
660 WARN_ON_ONCE(ret);
661 rtnl_unlock();
662}
663
664static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog,
665 struct bpf_prog *oprog)
666{
667 struct netkit_link *nkl = netkit_link(link);
668 struct bpf_mprog_entry *entry, *entry_new;
669 struct net_device *dev;
670 int ret = 0;
671
672 rtnl_lock();
673 dev = nkl->dev;
674 if (!dev) {
675 ret = -ENOLINK;
676 goto out;
677 }
678 if (oprog && link->prog != oprog) {
679 ret = -EPERM;
680 goto out;
681 }
682 oprog = link->prog;
683 if (oprog == nprog) {
684 bpf_prog_put(nprog);
685 goto out;
686 }
687 entry = netkit_entry_fetch(dev, false);
688 if (!entry) {
689 ret = -ENOENT;
690 goto out;
691 }
692 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
693 BPF_F_REPLACE | BPF_F_ID,
694 link->prog->aux->id, 0);
695 if (!ret) {
696 WARN_ON_ONCE(entry != entry_new);
697 oprog = xchg(&link->prog, nprog);
698 bpf_prog_put(oprog);
699 bpf_mprog_commit(entry);
700 }
701out:
702 rtnl_unlock();
703 return ret;
704}
705
706static void netkit_link_dealloc(struct bpf_link *link)
707{
708 kfree(netkit_link(link));
709}
710
711static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
712{
713 const struct netkit_link *nkl = netkit_link(link);
714 u32 ifindex = 0;
715
716 rtnl_lock();
717 if (nkl->dev)
718 ifindex = nkl->dev->ifindex;
719 rtnl_unlock();
720
721 seq_printf(seq, "ifindex:\t%u\n", ifindex);
722 seq_printf(seq, "attach_type:\t%u (%s)\n",
723 nkl->location,
724 nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
725}
726
727static int netkit_link_fill_info(const struct bpf_link *link,
728 struct bpf_link_info *info)
729{
730 const struct netkit_link *nkl = netkit_link(link);
731 u32 ifindex = 0;
732
733 rtnl_lock();
734 if (nkl->dev)
735 ifindex = nkl->dev->ifindex;
736 rtnl_unlock();
737
738 info->netkit.ifindex = ifindex;
739 info->netkit.attach_type = nkl->location;
740 return 0;
741}
742
743static int netkit_link_detach(struct bpf_link *link)
744{
745 netkit_link_release(link);
746 return 0;
747}
748
749static const struct bpf_link_ops netkit_link_lops = {
750 .release = netkit_link_release,
751 .detach = netkit_link_detach,
752 .dealloc = netkit_link_dealloc,
753 .update_prog = netkit_link_update,
754 .show_fdinfo = netkit_link_fdinfo,
755 .fill_link_info = netkit_link_fill_info,
756};
757
758static int netkit_link_init(struct netkit_link *nkl,
759 struct bpf_link_primer *link_primer,
760 const union bpf_attr *attr,
761 struct net_device *dev,
762 struct bpf_prog *prog)
763{
764 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
765 &netkit_link_lops, prog);
766 nkl->location = attr->link_create.attach_type;
767 nkl->dev = dev;
768 return bpf_link_prime(&nkl->link, link_primer);
769}
770
771int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
772{
773 struct bpf_link_primer link_primer;
774 struct netkit_link *nkl;
775 struct net_device *dev;
776 int ret;
777
778 rtnl_lock();
779 dev = netkit_dev_fetch(current->nsproxy->net_ns,
780 attr->link_create.target_ifindex,
781 attr->link_create.attach_type);
782 if (IS_ERR(dev)) {
783 ret = PTR_ERR(dev);
784 goto out;
785 }
786 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT);
787 if (!nkl) {
788 ret = -ENOMEM;
789 goto out;
790 }
791 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog);
792 if (ret) {
793 kfree(nkl);
794 goto out;
795 }
796 ret = netkit_link_prog_attach(&nkl->link,
797 attr->link_create.flags,
798 attr->link_create.netkit.relative_fd,
799 attr->link_create.netkit.expected_revision);
800 if (ret) {
801 nkl->dev = NULL;
802 bpf_link_cleanup(&link_primer);
803 goto out;
804 }
805 ret = bpf_link_settle(&link_primer);
806out:
807 rtnl_unlock();
808 return ret;
809}
810
811static void netkit_release_all(struct net_device *dev)
812{
813 struct bpf_mprog_entry *entry;
814 struct bpf_tuple tuple = {};
815 struct bpf_mprog_fp *fp;
816 struct bpf_mprog_cp *cp;
817
818 entry = netkit_entry_fetch(dev, false);
819 if (!entry)
820 return;
821 netkit_entry_update(dev, NULL);
822 netkit_entry_sync();
823 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
824 if (tuple.link)
825 netkit_link(tuple.link)->dev = NULL;
826 else
827 bpf_prog_put(tuple.prog);
828 }
829}
830
831static void netkit_uninit(struct net_device *dev)
832{
833 netkit_release_all(dev);
834}
835
836static void netkit_del_link(struct net_device *dev, struct list_head *head)
837{
838 struct netkit *nk = netkit_priv(dev);
839 struct net_device *peer = rtnl_dereference(nk->peer);
840
841 RCU_INIT_POINTER(nk->peer, NULL);
842 unregister_netdevice_queue(dev, head);
843 if (peer) {
844 nk = netkit_priv(peer);
845 RCU_INIT_POINTER(nk->peer, NULL);
846 unregister_netdevice_queue(peer, head);
847 }
848}
849
850static int netkit_change_link(struct net_device *dev, struct nlattr *tb[],
851 struct nlattr *data[],
852 struct netlink_ext_ack *extack)
853{
854 struct netkit *nk = netkit_priv(dev);
855 struct net_device *peer = rtnl_dereference(nk->peer);
856 enum netkit_action policy;
857 struct nlattr *attr;
858 int err;
859
860 if (!nk->primary) {
861 NL_SET_ERR_MSG(extack,
862 "netkit link settings can be changed only through the primary device");
863 return -EACCES;
864 }
865
866 if (data[IFLA_NETKIT_MODE]) {
867 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE],
868 "netkit link operating mode cannot be changed after device creation");
869 return -EACCES;
870 }
871
872 if (data[IFLA_NETKIT_PEER_INFO]) {
873 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO],
874 "netkit peer info cannot be changed after device creation");
875 return -EINVAL;
876 }
877
878 if (data[IFLA_NETKIT_POLICY]) {
879 attr = data[IFLA_NETKIT_POLICY];
880 policy = nla_get_u32(attr);
881 err = netkit_check_policy(policy, attr, extack);
882 if (err)
883 return err;
884 WRITE_ONCE(nk->policy, policy);
885 }
886
887 if (data[IFLA_NETKIT_PEER_POLICY]) {
888 err = -EOPNOTSUPP;
889 attr = data[IFLA_NETKIT_PEER_POLICY];
890 policy = nla_get_u32(attr);
891 if (peer)
892 err = netkit_check_policy(policy, attr, extack);
893 if (err)
894 return err;
895 nk = netkit_priv(peer);
896 WRITE_ONCE(nk->policy, policy);
897 }
898
899 return 0;
900}
901
902static size_t netkit_get_size(const struct net_device *dev)
903{
904 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */
905 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */
906 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */
907 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */
908 0;
909}
910
911static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
912{
913 struct netkit *nk = netkit_priv(dev);
914 struct net_device *peer = rtnl_dereference(nk->peer);
915
916 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
917 return -EMSGSIZE;
918 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
919 return -EMSGSIZE;
920 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
921 return -EMSGSIZE;
922
923 if (peer) {
924 nk = netkit_priv(peer);
925 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
926 return -EMSGSIZE;
927 }
928
929 return 0;
930}
931
932static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = {
933 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) },
934 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 },
935 [IFLA_NETKIT_MODE] = { .type = NLA_U32 },
936 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 },
937 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT,
938 .reject_message = "Primary attribute is read-only" },
939};
940
941static struct rtnl_link_ops netkit_link_ops = {
942 .kind = DRV_NAME,
943 .priv_size = sizeof(struct netkit),
944 .setup = netkit_setup,
945 .newlink = netkit_new_link,
946 .dellink = netkit_del_link,
947 .changelink = netkit_change_link,
948 .get_link_net = netkit_get_link_net,
949 .get_size = netkit_get_size,
950 .fill_info = netkit_fill_info,
951 .policy = netkit_policy,
952 .validate = netkit_validate,
953 .maxtype = IFLA_NETKIT_MAX,
954};
955
956static __init int netkit_init(void)
957{
958 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT ||
959 (int)NETKIT_PASS != (int)TCX_PASS ||
960 (int)NETKIT_DROP != (int)TCX_DROP ||
961 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
962
963 return rtnl_link_register(&netkit_link_ops);
964}
965
966static __exit void netkit_exit(void)
967{
968 rtnl_link_unregister(&netkit_link_ops);
969}
970
971module_init(netkit_init);
972module_exit(netkit_exit);
973
974MODULE_DESCRIPTION("BPF-programmable network device");
975MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");
976MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");
977MODULE_LICENSE("GPL");
978MODULE_ALIAS_RTNL_LINK(DRV_NAME);