Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 */
5
6#include "queueing.h"
7#include "socket.h"
8#include "timers.h"
9#include "device.h"
10#include "ratelimiter.h"
11#include "peer.h"
12#include "messages.h"
13
14#include <linux/module.h>
15#include <linux/rtnetlink.h>
16#include <linux/inet.h>
17#include <linux/netdevice.h>
18#include <linux/inetdevice.h>
19#include <linux/if_arp.h>
20#include <linux/icmp.h>
21#include <linux/suspend.h>
22#include <net/dst_metadata.h>
23#include <net/icmp.h>
24#include <net/rtnetlink.h>
25#include <net/ip_tunnels.h>
26#include <net/addrconf.h>
27
28static LIST_HEAD(device_list);
29
30static int wg_open(struct net_device *dev)
31{
32 struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
33 struct inet6_dev *dev_v6 = __in6_dev_get(dev);
34 struct wg_device *wg = netdev_priv(dev);
35 struct wg_peer *peer;
36 int ret;
37
38 if (dev_v4) {
39 /* At some point we might put this check near the ip_rt_send_
40 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
41 * to the current secpath check.
42 */
43 IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
44 IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
45 }
46 if (dev_v6)
47 dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
48
49 mutex_lock(&wg->device_update_lock);
50 ret = wg_socket_init(wg, wg->incoming_port);
51 if (ret < 0)
52 goto out;
53 list_for_each_entry(peer, &wg->peer_list, peer_list) {
54 wg_packet_send_staged_packets(peer);
55 if (peer->persistent_keepalive_interval)
56 wg_packet_send_keepalive(peer);
57 }
58out:
59 mutex_unlock(&wg->device_update_lock);
60 return ret;
61}
62
63static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
64{
65 struct wg_device *wg;
66 struct wg_peer *peer;
67
68 /* If the machine is constantly suspending and resuming, as part of
69 * its normal operation rather than as a somewhat rare event, then we
70 * don't actually want to clear keys.
71 */
72 if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) ||
73 IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP))
74 return 0;
75
76 if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
77 return 0;
78
79 rtnl_lock();
80 list_for_each_entry(wg, &device_list, device_list) {
81 mutex_lock(&wg->device_update_lock);
82 list_for_each_entry(peer, &wg->peer_list, peer_list) {
83 del_timer(&peer->timer_zero_key_material);
84 wg_noise_handshake_clear(&peer->handshake);
85 wg_noise_keypairs_clear(&peer->keypairs);
86 }
87 mutex_unlock(&wg->device_update_lock);
88 }
89 rtnl_unlock();
90 rcu_barrier();
91 return 0;
92}
93
94static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
95
96static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data)
97{
98 struct wg_device *wg;
99 struct wg_peer *peer;
100
101 rtnl_lock();
102 list_for_each_entry(wg, &device_list, device_list) {
103 mutex_lock(&wg->device_update_lock);
104 list_for_each_entry(peer, &wg->peer_list, peer_list)
105 wg_noise_expire_current_peer_keypairs(peer);
106 mutex_unlock(&wg->device_update_lock);
107 }
108 rtnl_unlock();
109 return 0;
110}
111
112static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification };
113
114static int wg_stop(struct net_device *dev)
115{
116 struct wg_device *wg = netdev_priv(dev);
117 struct wg_peer *peer;
118 struct sk_buff *skb;
119
120 mutex_lock(&wg->device_update_lock);
121 list_for_each_entry(peer, &wg->peer_list, peer_list) {
122 wg_packet_purge_staged_packets(peer);
123 wg_timers_stop(peer);
124 wg_noise_handshake_clear(&peer->handshake);
125 wg_noise_keypairs_clear(&peer->keypairs);
126 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
127 }
128 mutex_unlock(&wg->device_update_lock);
129 while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
130 kfree_skb(skb);
131 atomic_set(&wg->handshake_queue_len, 0);
132 wg_socket_reinit(wg, NULL, NULL);
133 return 0;
134}
135
136static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
137{
138 struct wg_device *wg = netdev_priv(dev);
139 struct sk_buff_head packets;
140 struct wg_peer *peer;
141 struct sk_buff *next;
142 sa_family_t family;
143 u32 mtu;
144 int ret;
145
146 if (unlikely(!wg_check_packet_protocol(skb))) {
147 ret = -EPROTONOSUPPORT;
148 net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
149 goto err;
150 }
151
152 peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
153 if (unlikely(!peer)) {
154 ret = -ENOKEY;
155 if (skb->protocol == htons(ETH_P_IP))
156 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
157 dev->name, &ip_hdr(skb)->daddr);
158 else if (skb->protocol == htons(ETH_P_IPV6))
159 net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
160 dev->name, &ipv6_hdr(skb)->daddr);
161 goto err_icmp;
162 }
163
164 family = READ_ONCE(peer->endpoint.addr.sa_family);
165 if (unlikely(family != AF_INET && family != AF_INET6)) {
166 ret = -EDESTADDRREQ;
167 net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
168 dev->name, peer->internal_id);
169 goto err_peer;
170 }
171
172 mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
173
174 __skb_queue_head_init(&packets);
175 if (!skb_is_gso(skb)) {
176 skb_mark_not_on_list(skb);
177 } else {
178 struct sk_buff *segs = skb_gso_segment(skb, 0);
179
180 if (IS_ERR(segs)) {
181 ret = PTR_ERR(segs);
182 goto err_peer;
183 }
184 dev_kfree_skb(skb);
185 skb = segs;
186 }
187
188 skb_list_walk_safe(skb, skb, next) {
189 skb_mark_not_on_list(skb);
190
191 skb = skb_share_check(skb, GFP_ATOMIC);
192 if (unlikely(!skb))
193 continue;
194
195 /* We only need to keep the original dst around for icmp,
196 * so at this point we're in a position to drop it.
197 */
198 skb_dst_drop(skb);
199
200 PACKET_CB(skb)->mtu = mtu;
201
202 __skb_queue_tail(&packets, skb);
203 }
204
205 spin_lock_bh(&peer->staged_packet_queue.lock);
206 /* If the queue is getting too big, we start removing the oldest packets
207 * until it's small again. We do this before adding the new packet, so
208 * we don't remove GSO segments that are in excess.
209 */
210 while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
211 dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
212 ++dev->stats.tx_dropped;
213 }
214 skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
215 spin_unlock_bh(&peer->staged_packet_queue.lock);
216
217 wg_packet_send_staged_packets(peer);
218
219 wg_peer_put(peer);
220 return NETDEV_TX_OK;
221
222err_peer:
223 wg_peer_put(peer);
224err_icmp:
225 if (skb->protocol == htons(ETH_P_IP))
226 icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
227 else if (skb->protocol == htons(ETH_P_IPV6))
228 icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
229err:
230 ++dev->stats.tx_errors;
231 kfree_skb(skb);
232 return ret;
233}
234
235static const struct net_device_ops netdev_ops = {
236 .ndo_open = wg_open,
237 .ndo_stop = wg_stop,
238 .ndo_start_xmit = wg_xmit,
239 .ndo_get_stats64 = dev_get_tstats64
240};
241
242static void wg_destruct(struct net_device *dev)
243{
244 struct wg_device *wg = netdev_priv(dev);
245
246 rtnl_lock();
247 list_del(&wg->device_list);
248 rtnl_unlock();
249 mutex_lock(&wg->device_update_lock);
250 rcu_assign_pointer(wg->creating_net, NULL);
251 wg->incoming_port = 0;
252 wg_socket_reinit(wg, NULL, NULL);
253 /* The final references are cleared in the below calls to destroy_workqueue. */
254 wg_peer_remove_all(wg);
255 destroy_workqueue(wg->handshake_receive_wq);
256 destroy_workqueue(wg->handshake_send_wq);
257 destroy_workqueue(wg->packet_crypt_wq);
258 wg_packet_queue_free(&wg->handshake_queue, true);
259 wg_packet_queue_free(&wg->decrypt_queue, false);
260 wg_packet_queue_free(&wg->encrypt_queue, false);
261 rcu_barrier(); /* Wait for all the peers to be actually freed. */
262 wg_ratelimiter_uninit();
263 memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
264 free_percpu(dev->tstats);
265 kvfree(wg->index_hashtable);
266 kvfree(wg->peer_hashtable);
267 mutex_unlock(&wg->device_update_lock);
268
269 pr_debug("%s: Interface destroyed\n", dev->name);
270 free_netdev(dev);
271}
272
273static const struct device_type device_type = { .name = KBUILD_MODNAME };
274
275static void wg_setup(struct net_device *dev)
276{
277 struct wg_device *wg = netdev_priv(dev);
278 enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
279 NETIF_F_SG | NETIF_F_GSO |
280 NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
281 const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
282 max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
283
284 dev->netdev_ops = &netdev_ops;
285 dev->header_ops = &ip_tunnel_header_ops;
286 dev->hard_header_len = 0;
287 dev->addr_len = 0;
288 dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
289 dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
290 dev->type = ARPHRD_NONE;
291 dev->flags = IFF_POINTOPOINT | IFF_NOARP;
292 dev->priv_flags |= IFF_NO_QUEUE;
293 dev->features |= NETIF_F_LLTX;
294 dev->features |= WG_NETDEV_FEATURES;
295 dev->hw_features |= WG_NETDEV_FEATURES;
296 dev->hw_enc_features |= WG_NETDEV_FEATURES;
297 dev->mtu = ETH_DATA_LEN - overhead;
298 dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
299
300 SET_NETDEV_DEVTYPE(dev, &device_type);
301
302 /* We need to keep the dst around in case of icmp replies. */
303 netif_keep_dst(dev);
304
305 memset(wg, 0, sizeof(*wg));
306 wg->dev = dev;
307}
308
309static int wg_newlink(struct net *src_net, struct net_device *dev,
310 struct nlattr *tb[], struct nlattr *data[],
311 struct netlink_ext_ack *extack)
312{
313 struct wg_device *wg = netdev_priv(dev);
314 int ret = -ENOMEM;
315
316 rcu_assign_pointer(wg->creating_net, src_net);
317 init_rwsem(&wg->static_identity.lock);
318 mutex_init(&wg->socket_update_lock);
319 mutex_init(&wg->device_update_lock);
320 wg_allowedips_init(&wg->peer_allowedips);
321 wg_cookie_checker_init(&wg->cookie_checker, wg);
322 INIT_LIST_HEAD(&wg->peer_list);
323 wg->device_update_gen = 1;
324
325 wg->peer_hashtable = wg_pubkey_hashtable_alloc();
326 if (!wg->peer_hashtable)
327 return ret;
328
329 wg->index_hashtable = wg_index_hashtable_alloc();
330 if (!wg->index_hashtable)
331 goto err_free_peer_hashtable;
332
333 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
334 if (!dev->tstats)
335 goto err_free_index_hashtable;
336
337 wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
338 WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
339 if (!wg->handshake_receive_wq)
340 goto err_free_tstats;
341
342 wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
343 WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
344 if (!wg->handshake_send_wq)
345 goto err_destroy_handshake_receive;
346
347 wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
348 WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
349 if (!wg->packet_crypt_wq)
350 goto err_destroy_handshake_send;
351
352 ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
353 MAX_QUEUED_PACKETS);
354 if (ret < 0)
355 goto err_destroy_packet_crypt;
356
357 ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
358 MAX_QUEUED_PACKETS);
359 if (ret < 0)
360 goto err_free_encrypt_queue;
361
362 ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
363 MAX_QUEUED_INCOMING_HANDSHAKES);
364 if (ret < 0)
365 goto err_free_decrypt_queue;
366
367 ret = wg_ratelimiter_init();
368 if (ret < 0)
369 goto err_free_handshake_queue;
370
371 ret = register_netdevice(dev);
372 if (ret < 0)
373 goto err_uninit_ratelimiter;
374
375 list_add(&wg->device_list, &device_list);
376
377 /* We wait until the end to assign priv_destructor, so that
378 * register_netdevice doesn't call it for us if it fails.
379 */
380 dev->priv_destructor = wg_destruct;
381
382 pr_debug("%s: Interface created\n", dev->name);
383 return ret;
384
385err_uninit_ratelimiter:
386 wg_ratelimiter_uninit();
387err_free_handshake_queue:
388 wg_packet_queue_free(&wg->handshake_queue, false);
389err_free_decrypt_queue:
390 wg_packet_queue_free(&wg->decrypt_queue, false);
391err_free_encrypt_queue:
392 wg_packet_queue_free(&wg->encrypt_queue, false);
393err_destroy_packet_crypt:
394 destroy_workqueue(wg->packet_crypt_wq);
395err_destroy_handshake_send:
396 destroy_workqueue(wg->handshake_send_wq);
397err_destroy_handshake_receive:
398 destroy_workqueue(wg->handshake_receive_wq);
399err_free_tstats:
400 free_percpu(dev->tstats);
401err_free_index_hashtable:
402 kvfree(wg->index_hashtable);
403err_free_peer_hashtable:
404 kvfree(wg->peer_hashtable);
405 return ret;
406}
407
408static struct rtnl_link_ops link_ops __read_mostly = {
409 .kind = KBUILD_MODNAME,
410 .priv_size = sizeof(struct wg_device),
411 .setup = wg_setup,
412 .newlink = wg_newlink,
413};
414
415static void wg_netns_pre_exit(struct net *net)
416{
417 struct wg_device *wg;
418 struct wg_peer *peer;
419
420 rtnl_lock();
421 list_for_each_entry(wg, &device_list, device_list) {
422 if (rcu_access_pointer(wg->creating_net) == net) {
423 pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
424 netif_carrier_off(wg->dev);
425 mutex_lock(&wg->device_update_lock);
426 rcu_assign_pointer(wg->creating_net, NULL);
427 wg_socket_reinit(wg, NULL, NULL);
428 list_for_each_entry(peer, &wg->peer_list, peer_list)
429 wg_socket_clear_peer_endpoint_src(peer);
430 mutex_unlock(&wg->device_update_lock);
431 }
432 }
433 rtnl_unlock();
434}
435
436static struct pernet_operations pernet_ops = {
437 .pre_exit = wg_netns_pre_exit
438};
439
440int __init wg_device_init(void)
441{
442 int ret;
443
444 ret = register_pm_notifier(&pm_notifier);
445 if (ret)
446 return ret;
447
448 ret = register_random_vmfork_notifier(&vm_notifier);
449 if (ret)
450 goto error_pm;
451
452 ret = register_pernet_device(&pernet_ops);
453 if (ret)
454 goto error_vm;
455
456 ret = rtnl_link_register(&link_ops);
457 if (ret)
458 goto error_pernet;
459
460 return 0;
461
462error_pernet:
463 unregister_pernet_device(&pernet_ops);
464error_vm:
465 unregister_random_vmfork_notifier(&vm_notifier);
466error_pm:
467 unregister_pm_notifier(&pm_notifier);
468 return ret;
469}
470
471void wg_device_uninit(void)
472{
473 rtnl_link_unregister(&link_ops);
474 unregister_pernet_device(&pernet_ops);
475 unregister_random_vmfork_notifier(&vm_notifier);
476 unregister_pm_notifier(&pm_notifier);
477 rcu_barrier();
478}