Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 */
5
6#include "peer.h"
7#include "device.h"
8#include "queueing.h"
9#include "timers.h"
10#include "peerlookup.h"
11#include "noise.h"
12
13#include <linux/kref.h>
14#include <linux/lockdep.h>
15#include <linux/rcupdate.h>
16#include <linux/list.h>
17
18static struct kmem_cache *peer_cache;
19static atomic64_t peer_counter = ATOMIC64_INIT(0);
20
21struct wg_peer *wg_peer_create(struct wg_device *wg,
22 const u8 public_key[NOISE_PUBLIC_KEY_LEN],
23 const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
24{
25 struct wg_peer *peer;
26 int ret = -ENOMEM;
27
28 lockdep_assert_held(&wg->device_update_lock);
29
30 if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
31 return ERR_PTR(ret);
32
33 peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL);
34 if (unlikely(!peer))
35 return ERR_PTR(ret);
36 if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)))
37 goto err;
38
39 peer->device = wg;
40 wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
41 public_key, preshared_key, peer);
42 peer->internal_id = atomic64_inc_return(&peer_counter);
43 peer->serial_work_cpu = nr_cpumask_bits;
44 wg_cookie_init(&peer->latest_cookie);
45 wg_timers_init(peer);
46 wg_cookie_checker_precompute_peer_keys(peer);
47 spin_lock_init(&peer->keypairs.keypair_update_lock);
48 INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker);
49 INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker);
50 wg_prev_queue_init(&peer->tx_queue);
51 wg_prev_queue_init(&peer->rx_queue);
52 rwlock_init(&peer->endpoint_lock);
53 kref_init(&peer->refcount);
54 skb_queue_head_init(&peer->staged_packet_queue);
55 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
56 set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
57 netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll);
58 napi_enable(&peer->napi);
59 list_add_tail(&peer->peer_list, &wg->peer_list);
60 INIT_LIST_HEAD(&peer->allowedips_list);
61 wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
62 ++wg->num_peers;
63 pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
64 return peer;
65
66err:
67 kmem_cache_free(peer_cache, peer);
68 return ERR_PTR(ret);
69}
70
71struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
72{
73 RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
74 "Taking peer reference without holding the RCU read lock");
75 if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
76 return NULL;
77 return peer;
78}
79
80static void peer_make_dead(struct wg_peer *peer)
81{
82 /* Remove from configuration-time lookup structures. */
83 list_del_init(&peer->peer_list);
84 wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
85 &peer->device->device_update_lock);
86 wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
87
88 /* Mark as dead, so that we don't allow jumping contexts after. */
89 WRITE_ONCE(peer->is_dead, true);
90
91 /* The caller must now synchronize_net() for this to take effect. */
92}
93
94static void peer_remove_after_dead(struct wg_peer *peer)
95{
96 WARN_ON(!peer->is_dead);
97
98 /* No more keypairs can be created for this peer, since is_dead protects
99 * add_new_keypair, so we can now destroy existing ones.
100 */
101 wg_noise_keypairs_clear(&peer->keypairs);
102
103 /* Destroy all ongoing timers that were in-flight at the beginning of
104 * this function.
105 */
106 wg_timers_stop(peer);
107
108 /* The transition between packet encryption/decryption queues isn't
109 * guarded by is_dead, but each reference's life is strictly bounded by
110 * two generations: once for parallel crypto and once for serial
111 * ingestion, so we can simply flush twice, and be sure that we no
112 * longer have references inside these queues.
113 */
114
115 /* a) For encrypt/decrypt. */
116 flush_workqueue(peer->device->packet_crypt_wq);
117 /* b.1) For send (but not receive, since that's napi). */
118 flush_workqueue(peer->device->packet_crypt_wq);
119 /* b.2.1) For receive (but not send, since that's wq). */
120 napi_disable(&peer->napi);
121 /* b.2.1) It's now safe to remove the napi struct, which must be done
122 * here from process context.
123 */
124 netif_napi_del(&peer->napi);
125
126 /* Ensure any workstructs we own (like transmit_handshake_work or
127 * clear_peer_work) no longer are in use.
128 */
129 flush_workqueue(peer->device->handshake_send_wq);
130
131 /* After the above flushes, a peer might still be active in a few
132 * different contexts: 1) from xmit(), before hitting is_dead and
133 * returning, 2) from wg_packet_consume_data(), before hitting is_dead
134 * and returning, 3) from wg_receive_handshake_packet() after a point
135 * where it has processed an incoming handshake packet, but where
136 * all calls to pass it off to timers fails because of is_dead. We won't
137 * have new references in (1) eventually, because we're removed from
138 * allowedips; we won't have new references in (2) eventually, because
139 * wg_index_hashtable_lookup will always return NULL, since we removed
140 * all existing keypairs and no more can be created; we won't have new
141 * references in (3) eventually, because we're removed from the pubkey
142 * hash table, which allows for a maximum of one handshake response,
143 * via the still-uncleared index hashtable entry, but not more than one,
144 * and in wg_cookie_message_consume, the lookup eventually gets a peer
145 * with a refcount of zero, so no new reference is taken.
146 */
147
148 --peer->device->num_peers;
149 wg_peer_put(peer);
150}
151
152/* We have a separate "remove" function make sure that all active places where
153 * a peer is currently operating will eventually come to an end and not pass
154 * their reference onto another context.
155 */
156void wg_peer_remove(struct wg_peer *peer)
157{
158 if (unlikely(!peer))
159 return;
160 lockdep_assert_held(&peer->device->device_update_lock);
161
162 peer_make_dead(peer);
163 synchronize_net();
164 peer_remove_after_dead(peer);
165}
166
167void wg_peer_remove_all(struct wg_device *wg)
168{
169 struct wg_peer *peer, *temp;
170 LIST_HEAD(dead_peers);
171
172 lockdep_assert_held(&wg->device_update_lock);
173
174 /* Avoid having to traverse individually for each one. */
175 wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
176
177 list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
178 peer_make_dead(peer);
179 list_add_tail(&peer->peer_list, &dead_peers);
180 }
181 synchronize_net();
182 list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
183 peer_remove_after_dead(peer);
184}
185
186static void rcu_release(struct rcu_head *rcu)
187{
188 struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
189
190 dst_cache_destroy(&peer->endpoint_cache);
191 WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue));
192
193 /* The final zeroing takes care of clearing any remaining handshake key
194 * material and other potentially sensitive information.
195 */
196 memzero_explicit(peer, sizeof(*peer));
197 kmem_cache_free(peer_cache, peer);
198}
199
200static void kref_release(struct kref *refcount)
201{
202 struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
203
204 pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
205 peer->device->dev->name, peer->internal_id,
206 &peer->endpoint.addr);
207
208 /* Remove ourself from dynamic runtime lookup structures, now that the
209 * last reference is gone.
210 */
211 wg_index_hashtable_remove(peer->device->index_hashtable,
212 &peer->handshake.entry);
213
214 /* Remove any lingering packets that didn't have a chance to be
215 * transmitted.
216 */
217 wg_packet_purge_staged_packets(peer);
218
219 /* Free the memory used. */
220 call_rcu(&peer->rcu, rcu_release);
221}
222
223void wg_peer_put(struct wg_peer *peer)
224{
225 if (unlikely(!peer))
226 return;
227 kref_put(&peer->refcount, kref_release);
228}
229
230int __init wg_peer_init(void)
231{
232 peer_cache = KMEM_CACHE(wg_peer, 0);
233 return peer_cache ? 0 : -ENOMEM;
234}
235
236void wg_peer_uninit(void)
237{
238 kmem_cache_destroy(peer_cache);
239}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 */
5
6#include "peer.h"
7#include "device.h"
8#include "queueing.h"
9#include "timers.h"
10#include "peerlookup.h"
11#include "noise.h"
12
13#include <linux/kref.h>
14#include <linux/lockdep.h>
15#include <linux/rcupdate.h>
16#include <linux/list.h>
17
18static atomic64_t peer_counter = ATOMIC64_INIT(0);
19
20struct wg_peer *wg_peer_create(struct wg_device *wg,
21 const u8 public_key[NOISE_PUBLIC_KEY_LEN],
22 const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
23{
24 struct wg_peer *peer;
25 int ret = -ENOMEM;
26
27 lockdep_assert_held(&wg->device_update_lock);
28
29 if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
30 return ERR_PTR(ret);
31
32 peer = kzalloc(sizeof(*peer), GFP_KERNEL);
33 if (unlikely(!peer))
34 return ERR_PTR(ret);
35 peer->device = wg;
36
37 wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
38 public_key, preshared_key, peer);
39 if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))
40 goto err_1;
41 if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false,
42 MAX_QUEUED_PACKETS))
43 goto err_2;
44 if (wg_packet_queue_init(&peer->rx_queue, NULL, false,
45 MAX_QUEUED_PACKETS))
46 goto err_3;
47
48 peer->internal_id = atomic64_inc_return(&peer_counter);
49 peer->serial_work_cpu = nr_cpumask_bits;
50 wg_cookie_init(&peer->latest_cookie);
51 wg_timers_init(peer);
52 wg_cookie_checker_precompute_peer_keys(peer);
53 spin_lock_init(&peer->keypairs.keypair_update_lock);
54 INIT_WORK(&peer->transmit_handshake_work,
55 wg_packet_handshake_send_worker);
56 rwlock_init(&peer->endpoint_lock);
57 kref_init(&peer->refcount);
58 skb_queue_head_init(&peer->staged_packet_queue);
59 wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
60 set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
61 netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
62 NAPI_POLL_WEIGHT);
63 napi_enable(&peer->napi);
64 list_add_tail(&peer->peer_list, &wg->peer_list);
65 INIT_LIST_HEAD(&peer->allowedips_list);
66 wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
67 ++wg->num_peers;
68 pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
69 return peer;
70
71err_3:
72 wg_packet_queue_free(&peer->tx_queue, false);
73err_2:
74 dst_cache_destroy(&peer->endpoint_cache);
75err_1:
76 kfree(peer);
77 return ERR_PTR(ret);
78}
79
80struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
81{
82 RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
83 "Taking peer reference without holding the RCU read lock");
84 if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
85 return NULL;
86 return peer;
87}
88
89static void peer_make_dead(struct wg_peer *peer)
90{
91 /* Remove from configuration-time lookup structures. */
92 list_del_init(&peer->peer_list);
93 wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
94 &peer->device->device_update_lock);
95 wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
96
97 /* Mark as dead, so that we don't allow jumping contexts after. */
98 WRITE_ONCE(peer->is_dead, true);
99
100 /* The caller must now synchronize_rcu() for this to take effect. */
101}
102
103static void peer_remove_after_dead(struct wg_peer *peer)
104{
105 WARN_ON(!peer->is_dead);
106
107 /* No more keypairs can be created for this peer, since is_dead protects
108 * add_new_keypair, so we can now destroy existing ones.
109 */
110 wg_noise_keypairs_clear(&peer->keypairs);
111
112 /* Destroy all ongoing timers that were in-flight at the beginning of
113 * this function.
114 */
115 wg_timers_stop(peer);
116
117 /* The transition between packet encryption/decryption queues isn't
118 * guarded by is_dead, but each reference's life is strictly bounded by
119 * two generations: once for parallel crypto and once for serial
120 * ingestion, so we can simply flush twice, and be sure that we no
121 * longer have references inside these queues.
122 */
123
124 /* a) For encrypt/decrypt. */
125 flush_workqueue(peer->device->packet_crypt_wq);
126 /* b.1) For send (but not receive, since that's napi). */
127 flush_workqueue(peer->device->packet_crypt_wq);
128 /* b.2.1) For receive (but not send, since that's wq). */
129 napi_disable(&peer->napi);
130 /* b.2.1) It's now safe to remove the napi struct, which must be done
131 * here from process context.
132 */
133 netif_napi_del(&peer->napi);
134
135 /* Ensure any workstructs we own (like transmit_handshake_work or
136 * clear_peer_work) no longer are in use.
137 */
138 flush_workqueue(peer->device->handshake_send_wq);
139
140 /* After the above flushes, a peer might still be active in a few
141 * different contexts: 1) from xmit(), before hitting is_dead and
142 * returning, 2) from wg_packet_consume_data(), before hitting is_dead
143 * and returning, 3) from wg_receive_handshake_packet() after a point
144 * where it has processed an incoming handshake packet, but where
145 * all calls to pass it off to timers fails because of is_dead. We won't
146 * have new references in (1) eventually, because we're removed from
147 * allowedips; we won't have new references in (2) eventually, because
148 * wg_index_hashtable_lookup will always return NULL, since we removed
149 * all existing keypairs and no more can be created; we won't have new
150 * references in (3) eventually, because we're removed from the pubkey
151 * hash table, which allows for a maximum of one handshake response,
152 * via the still-uncleared index hashtable entry, but not more than one,
153 * and in wg_cookie_message_consume, the lookup eventually gets a peer
154 * with a refcount of zero, so no new reference is taken.
155 */
156
157 --peer->device->num_peers;
158 wg_peer_put(peer);
159}
160
161/* We have a separate "remove" function make sure that all active places where
162 * a peer is currently operating will eventually come to an end and not pass
163 * their reference onto another context.
164 */
165void wg_peer_remove(struct wg_peer *peer)
166{
167 if (unlikely(!peer))
168 return;
169 lockdep_assert_held(&peer->device->device_update_lock);
170
171 peer_make_dead(peer);
172 synchronize_rcu();
173 peer_remove_after_dead(peer);
174}
175
176void wg_peer_remove_all(struct wg_device *wg)
177{
178 struct wg_peer *peer, *temp;
179 LIST_HEAD(dead_peers);
180
181 lockdep_assert_held(&wg->device_update_lock);
182
183 /* Avoid having to traverse individually for each one. */
184 wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
185
186 list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
187 peer_make_dead(peer);
188 list_add_tail(&peer->peer_list, &dead_peers);
189 }
190 synchronize_rcu();
191 list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
192 peer_remove_after_dead(peer);
193}
194
195static void rcu_release(struct rcu_head *rcu)
196{
197 struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
198
199 dst_cache_destroy(&peer->endpoint_cache);
200 wg_packet_queue_free(&peer->rx_queue, false);
201 wg_packet_queue_free(&peer->tx_queue, false);
202
203 /* The final zeroing takes care of clearing any remaining handshake key
204 * material and other potentially sensitive information.
205 */
206 kfree_sensitive(peer);
207}
208
209static void kref_release(struct kref *refcount)
210{
211 struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
212
213 pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
214 peer->device->dev->name, peer->internal_id,
215 &peer->endpoint.addr);
216
217 /* Remove ourself from dynamic runtime lookup structures, now that the
218 * last reference is gone.
219 */
220 wg_index_hashtable_remove(peer->device->index_hashtable,
221 &peer->handshake.entry);
222
223 /* Remove any lingering packets that didn't have a chance to be
224 * transmitted.
225 */
226 wg_packet_purge_staged_packets(peer);
227
228 /* Free the memory used. */
229 call_rcu(&peer->rcu, rcu_release);
230}
231
232void wg_peer_put(struct wg_peer *peer)
233{
234 if (unlikely(!peer))
235 return;
236 kref_put(&peer->refcount, kref_release);
237}