1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4 */
  5
  6#include "peer.h"
  7#include "device.h"
  8#include "queueing.h"
  9#include "timers.h"
 10#include "peerlookup.h"
 11#include "noise.h"
 12
 13#include <linux/kref.h>
 14#include <linux/lockdep.h>
 15#include <linux/rcupdate.h>
 16#include <linux/list.h>
 17
 18static atomic64_t peer_counter = ATOMIC64_INIT(0);
 19
 20struct wg_peer *wg_peer_create(struct wg_device *wg,
 21			       const u8 public_key[NOISE_PUBLIC_KEY_LEN],
 22			       const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
 23{
 24	struct wg_peer *peer;
 25	int ret = -ENOMEM;
 26
 27	lockdep_assert_held(&wg->device_update_lock);
 28
 29	if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
 30		return ERR_PTR(ret);
 31
 32	peer = kzalloc(sizeof(*peer), GFP_KERNEL);
 33	if (unlikely(!peer))
 34		return ERR_PTR(ret);
 35	peer->device = wg;
 36
 37	wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
 38				public_key, preshared_key, peer);
 39	if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))
 40		goto err_1;
 41	if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false,
 42				 MAX_QUEUED_PACKETS))
 43		goto err_2;
 44	if (wg_packet_queue_init(&peer->rx_queue, NULL, false,
 45				 MAX_QUEUED_PACKETS))
 46		goto err_3;
 47
 48	peer->internal_id = atomic64_inc_return(&peer_counter);
 49	peer->serial_work_cpu = nr_cpumask_bits;
 50	wg_cookie_init(&peer->latest_cookie);
 51	wg_timers_init(peer);
 52	wg_cookie_checker_precompute_peer_keys(peer);
 53	spin_lock_init(&peer->keypairs.keypair_update_lock);
 54	INIT_WORK(&peer->transmit_handshake_work,
 55		  wg_packet_handshake_send_worker);
 56	rwlock_init(&peer->endpoint_lock);
 57	kref_init(&peer->refcount);
 58	skb_queue_head_init(&peer->staged_packet_queue);
 59	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
 60	set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
 61	netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
 62		       NAPI_POLL_WEIGHT);
 63	napi_enable(&peer->napi);
 64	list_add_tail(&peer->peer_list, &wg->peer_list);
 65	INIT_LIST_HEAD(&peer->allowedips_list);
 66	wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
 67	++wg->num_peers;
 68	pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
 69	return peer;
 70
 71err_3:
 72	wg_packet_queue_free(&peer->tx_queue, false);
 73err_2:
 74	dst_cache_destroy(&peer->endpoint_cache);
 75err_1:
 76	kfree(peer);
 77	return ERR_PTR(ret);
 78}
 79
 80struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
 81{
 82	RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
 83			 "Taking peer reference without holding the RCU read lock");
 84	if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
 85		return NULL;
 86	return peer;
 87}
 88
 89static void peer_make_dead(struct wg_peer *peer)
 90{
 91	/* Remove from configuration-time lookup structures. */
 92	list_del_init(&peer->peer_list);
 93	wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
 94				     &peer->device->device_update_lock);
 95	wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
 96
 97	/* Mark as dead, so that we don't allow jumping contexts after. */
 98	WRITE_ONCE(peer->is_dead, true);
 99
100	/* The caller must now synchronize_rcu() for this to take effect. */
101}
102
103static void peer_remove_after_dead(struct wg_peer *peer)
104{
105	WARN_ON(!peer->is_dead);
106
107	/* No more keypairs can be created for this peer, since is_dead protects
108	 * add_new_keypair, so we can now destroy existing ones.
109	 */
110	wg_noise_keypairs_clear(&peer->keypairs);
111
112	/* Destroy all ongoing timers that were in-flight at the beginning of
113	 * this function.
114	 */
115	wg_timers_stop(peer);
116
117	/* The transition between packet encryption/decryption queues isn't
118	 * guarded by is_dead, but each reference's life is strictly bounded by
119	 * two generations: once for parallel crypto and once for serial
120	 * ingestion, so we can simply flush twice, and be sure that we no
121	 * longer have references inside these queues.
122	 */
123
124	/* a) For encrypt/decrypt. */
125	flush_workqueue(peer->device->packet_crypt_wq);
126	/* b.1) For send (but not receive, since that's napi). */
127	flush_workqueue(peer->device->packet_crypt_wq);
128	/* b.2.1) For receive (but not send, since that's wq). */
129	napi_disable(&peer->napi);
130	/* b.2.1) It's now safe to remove the napi struct, which must be done
131	 * here from process context.
132	 */
133	netif_napi_del(&peer->napi);
134
135	/* Ensure any workstructs we own (like transmit_handshake_work or
136	 * clear_peer_work) no longer are in use.
137	 */
138	flush_workqueue(peer->device->handshake_send_wq);
139
140	/* After the above flushes, a peer might still be active in a few
141	 * different contexts: 1) from xmit(), before hitting is_dead and
142	 * returning, 2) from wg_packet_consume_data(), before hitting is_dead
143	 * and returning, 3) from wg_receive_handshake_packet() after a point
144	 * where it has processed an incoming handshake packet, but where
145	 * all calls to pass it off to timers fails because of is_dead. We won't
146	 * have new references in (1) eventually, because we're removed from
147	 * allowedips; we won't have new references in (2) eventually, because
148	 * wg_index_hashtable_lookup will always return NULL, since we removed
149	 * all existing keypairs and no more can be created; we won't have new
150	 * references in (3) eventually, because we're removed from the pubkey
151	 * hash table, which allows for a maximum of one handshake response,
152	 * via the still-uncleared index hashtable entry, but not more than one,
153	 * and in wg_cookie_message_consume, the lookup eventually gets a peer
154	 * with a refcount of zero, so no new reference is taken.
155	 */
156
157	--peer->device->num_peers;
158	wg_peer_put(peer);
159}
160
161/* We have a separate "remove" function make sure that all active places where
162 * a peer is currently operating will eventually come to an end and not pass
163 * their reference onto another context.
164 */
165void wg_peer_remove(struct wg_peer *peer)
166{
167	if (unlikely(!peer))
168		return;
169	lockdep_assert_held(&peer->device->device_update_lock);
170
171	peer_make_dead(peer);
172	synchronize_rcu();
173	peer_remove_after_dead(peer);
174}
175
176void wg_peer_remove_all(struct wg_device *wg)
177{
178	struct wg_peer *peer, *temp;
179	LIST_HEAD(dead_peers);
180
181	lockdep_assert_held(&wg->device_update_lock);
182
183	/* Avoid having to traverse individually for each one. */
184	wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
185
186	list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
187		peer_make_dead(peer);
188		list_add_tail(&peer->peer_list, &dead_peers);
189	}
190	synchronize_rcu();
191	list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
192		peer_remove_after_dead(peer);
193}
194
195static void rcu_release(struct rcu_head *rcu)
196{
197	struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
198
199	dst_cache_destroy(&peer->endpoint_cache);
200	wg_packet_queue_free(&peer->rx_queue, false);
201	wg_packet_queue_free(&peer->tx_queue, false);
202
203	/* The final zeroing takes care of clearing any remaining handshake key
204	 * material and other potentially sensitive information.
205	 */
206	kfree_sensitive(peer);
207}
208
209static void kref_release(struct kref *refcount)
210{
211	struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
212
213	pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
214		 peer->device->dev->name, peer->internal_id,
215		 &peer->endpoint.addr);
216
217	/* Remove ourself from dynamic runtime lookup structures, now that the
218	 * last reference is gone.
219	 */
220	wg_index_hashtable_remove(peer->device->index_hashtable,
221				  &peer->handshake.entry);
222
223	/* Remove any lingering packets that didn't have a chance to be
224	 * transmitted.
225	 */
226	wg_packet_purge_staged_packets(peer);
227
228	/* Free the memory used. */
229	call_rcu(&peer->rcu, rcu_release);
230}
231
232void wg_peer_put(struct wg_peer *peer)
233{
234	if (unlikely(!peer))
235		return;
236	kref_put(&peer->refcount, kref_release);
237}