1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#include <crypto/hash.h>
  4#include <linux/cpu.h>
  5#include <linux/kref.h>
  6#include <linux/module.h>
  7#include <linux/mutex.h>
  8#include <linux/percpu.h>
  9#include <linux/workqueue.h>
 10#include <net/tcp.h>
 11
 12static size_t __scratch_size;
 13struct sigpool_scratch {
 14	local_lock_t bh_lock;
 15	void __rcu *pad;
 16};
 17
 18static DEFINE_PER_CPU(struct sigpool_scratch, sigpool_scratch) = {
 19	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
 20};
 21
 22struct sigpool_entry {
 23	struct crypto_ahash	*hash;
 24	const char		*alg;
 25	struct kref		kref;
 26	uint16_t		needs_key:1,
 27				reserved:15;
 28};
 29
 30#define CPOOL_SIZE (PAGE_SIZE / sizeof(struct sigpool_entry))
 31static struct sigpool_entry cpool[CPOOL_SIZE];
 32static unsigned int cpool_populated;
 33static DEFINE_MUTEX(cpool_mutex);
 34
 35/* Slow-path */
 36struct scratches_to_free {
 37	struct rcu_head rcu;
 38	unsigned int cnt;
 39	void *scratches[];
 40};
 41
 42static void free_old_scratches(struct rcu_head *head)
 43{
 44	struct scratches_to_free *stf;
 45
 46	stf = container_of(head, struct scratches_to_free, rcu);
 47	while (stf->cnt--)
 48		kfree(stf->scratches[stf->cnt]);
 49	kfree(stf);
 50}
 51
 52/**
 53 * sigpool_reserve_scratch - re-allocates scratch buffer, slow-path
 54 * @size: request size for the scratch/temp buffer
 55 */
 56static int sigpool_reserve_scratch(size_t size)
 57{
 58	struct scratches_to_free *stf;
 59	size_t stf_sz = struct_size(stf, scratches, num_possible_cpus());
 60	int cpu, err = 0;
 61
 62	lockdep_assert_held(&cpool_mutex);
 63	if (__scratch_size >= size)
 64		return 0;
 65
 66	stf = kmalloc(stf_sz, GFP_KERNEL);
 67	if (!stf)
 68		return -ENOMEM;
 69	stf->cnt = 0;
 70
 71	size = max(size, __scratch_size);
 72	cpus_read_lock();
 73	for_each_possible_cpu(cpu) {
 74		void *scratch, *old_scratch;
 75
 76		scratch = kmalloc_node(size, GFP_KERNEL, cpu_to_node(cpu));
 77		if (!scratch) {
 78			err = -ENOMEM;
 79			break;
 80		}
 81
 82		old_scratch = rcu_replace_pointer(per_cpu(sigpool_scratch.pad, cpu),
 83					scratch, lockdep_is_held(&cpool_mutex));
 84		if (!cpu_online(cpu) || !old_scratch) {
 85			kfree(old_scratch);
 86			continue;
 87		}
 88		stf->scratches[stf->cnt++] = old_scratch;
 89	}
 90	cpus_read_unlock();
 91	if (!err)
 92		__scratch_size = size;
 93
 94	call_rcu(&stf->rcu, free_old_scratches);
 95	return err;
 96}
 97
 98static void sigpool_scratch_free(void)
 99{
100	int cpu;
101
102	for_each_possible_cpu(cpu)
103		kfree(rcu_replace_pointer(per_cpu(sigpool_scratch.pad, cpu),
104					  NULL, lockdep_is_held(&cpool_mutex)));
105	__scratch_size = 0;
106}
107
108static int __cpool_try_clone(struct crypto_ahash *hash)
109{
110	struct crypto_ahash *tmp;
111
112	tmp = crypto_clone_ahash(hash);
113	if (IS_ERR(tmp))
114		return PTR_ERR(tmp);
115
116	crypto_free_ahash(tmp);
117	return 0;
118}
119
120static int __cpool_alloc_ahash(struct sigpool_entry *e, const char *alg)
121{
122	struct crypto_ahash *cpu0_hash;
123	int ret;
124
125	e->alg = kstrdup(alg, GFP_KERNEL);
126	if (!e->alg)
127		return -ENOMEM;
128
129	cpu0_hash = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC);
130	if (IS_ERR(cpu0_hash)) {
131		ret = PTR_ERR(cpu0_hash);
132		goto out_free_alg;
133	}
134
135	e->needs_key = crypto_ahash_get_flags(cpu0_hash) & CRYPTO_TFM_NEED_KEY;
136
137	ret = __cpool_try_clone(cpu0_hash);
138	if (ret)
139		goto out_free_cpu0_hash;
140	e->hash = cpu0_hash;
141	kref_init(&e->kref);
142	return 0;
143
144out_free_cpu0_hash:
145	crypto_free_ahash(cpu0_hash);
146out_free_alg:
147	kfree(e->alg);
148	e->alg = NULL;
149	return ret;
150}
151
152/**
153 * tcp_sigpool_alloc_ahash - allocates pool for ahash requests
154 * @alg: name of async hash algorithm
155 * @scratch_size: reserve a tcp_sigpool::scratch buffer of this size
156 */
157int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size)
158{
159	int i, ret;
160
161	/* slow-path */
162	mutex_lock(&cpool_mutex);
163	ret = sigpool_reserve_scratch(scratch_size);
164	if (ret)
165		goto out;
166	for (i = 0; i < cpool_populated; i++) {
167		if (!cpool[i].alg)
168			continue;
169		if (strcmp(cpool[i].alg, alg))
170			continue;
171
172		/* pairs with tcp_sigpool_release() */
173		if (!kref_get_unless_zero(&cpool[i].kref))
174			kref_init(&cpool[i].kref);
175		ret = i;
176		goto out;
177	}
178
179	for (i = 0; i < cpool_populated; i++) {
180		if (!cpool[i].alg)
181			break;
182	}
183	if (i >= CPOOL_SIZE) {
184		ret = -ENOSPC;
185		goto out;
186	}
187
188	ret = __cpool_alloc_ahash(&cpool[i], alg);
189	if (!ret) {
190		ret = i;
191		if (i == cpool_populated)
192			cpool_populated++;
193	}
194out:
195	mutex_unlock(&cpool_mutex);
196	return ret;
197}
198EXPORT_SYMBOL_GPL(tcp_sigpool_alloc_ahash);
199
200static void __cpool_free_entry(struct sigpool_entry *e)
201{
202	crypto_free_ahash(e->hash);
203	kfree(e->alg);
204	memset(e, 0, sizeof(*e));
205}
206
207static void cpool_cleanup_work_cb(struct work_struct *work)
208{
209	bool free_scratch = true;
210	unsigned int i;
211
212	mutex_lock(&cpool_mutex);
213	for (i = 0; i < cpool_populated; i++) {
214		if (kref_read(&cpool[i].kref) > 0) {
215			free_scratch = false;
216			continue;
217		}
218		if (!cpool[i].alg)
219			continue;
220		__cpool_free_entry(&cpool[i]);
221	}
222	if (free_scratch)
223		sigpool_scratch_free();
224	mutex_unlock(&cpool_mutex);
225}
226
227static DECLARE_WORK(cpool_cleanup_work, cpool_cleanup_work_cb);
228static void cpool_schedule_cleanup(struct kref *kref)
229{
230	schedule_work(&cpool_cleanup_work);
231}
232
233/**
234 * tcp_sigpool_release - decreases number of users for a pool. If it was
235 * the last user of the pool, releases any memory that was consumed.
236 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
237 */
238void tcp_sigpool_release(unsigned int id)
239{
240	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
241		return;
242
243	/* slow-path */
244	kref_put(&cpool[id].kref, cpool_schedule_cleanup);
245}
246EXPORT_SYMBOL_GPL(tcp_sigpool_release);
247
248/**
249 * tcp_sigpool_get - increases number of users (refcounter) for a pool
250 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
251 */
252void tcp_sigpool_get(unsigned int id)
253{
254	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
255		return;
256	kref_get(&cpool[id].kref);
257}
258EXPORT_SYMBOL_GPL(tcp_sigpool_get);
259
260int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(RCU_BH)
261{
262	struct crypto_ahash *hash;
263
264	rcu_read_lock_bh();
265	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg)) {
266		rcu_read_unlock_bh();
267		return -EINVAL;
268	}
269
270	hash = crypto_clone_ahash(cpool[id].hash);
271	if (IS_ERR(hash)) {
272		rcu_read_unlock_bh();
273		return PTR_ERR(hash);
274	}
275
276	c->req = ahash_request_alloc(hash, GFP_ATOMIC);
277	if (!c->req) {
278		crypto_free_ahash(hash);
279		rcu_read_unlock_bh();
280		return -ENOMEM;
281	}
282	ahash_request_set_callback(c->req, 0, NULL, NULL);
283
284	/* Pairs with tcp_sigpool_reserve_scratch(), scratch area is
285	 * valid (allocated) until tcp_sigpool_end().
286	 */
287	local_lock_nested_bh(&sigpool_scratch.bh_lock);
288	c->scratch = rcu_dereference_bh(*this_cpu_ptr(&sigpool_scratch.pad));
289	return 0;
290}
291EXPORT_SYMBOL_GPL(tcp_sigpool_start);
292
293void tcp_sigpool_end(struct tcp_sigpool *c) __releases(RCU_BH)
294{
295	struct crypto_ahash *hash = crypto_ahash_reqtfm(c->req);
296
297	local_unlock_nested_bh(&sigpool_scratch.bh_lock);
298	rcu_read_unlock_bh();
299	ahash_request_free(c->req);
300	crypto_free_ahash(hash);
301}
302EXPORT_SYMBOL_GPL(tcp_sigpool_end);
303
304/**
305 * tcp_sigpool_algo - return algorithm of tcp_sigpool
306 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
307 * @buf: buffer to return name of algorithm
308 * @buf_len: size of @buf
309 */
310size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len)
311{
312	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
313		return -EINVAL;
314
315	return strscpy(buf, cpool[id].alg, buf_len);
316}
317EXPORT_SYMBOL_GPL(tcp_sigpool_algo);
318
319/**
320 * tcp_sigpool_hash_skb_data - hash data in skb with initialized tcp_sigpool
321 * @hp: tcp_sigpool pointer
322 * @skb: buffer to add sign for
323 * @header_len: TCP header length for this segment
324 */
325int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
326			      const struct sk_buff *skb,
327			      unsigned int header_len)
328{
329	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
330					   skb_headlen(skb) - header_len : 0;
331	const struct skb_shared_info *shi = skb_shinfo(skb);
332	const struct tcphdr *tp = tcp_hdr(skb);
333	struct ahash_request *req = hp->req;
334	struct sk_buff *frag_iter;
335	struct scatterlist sg;
336	unsigned int i;
337
338	sg_init_table(&sg, 1);
339
340	sg_set_buf(&sg, ((u8 *)tp) + header_len, head_data_len);
341	ahash_request_set_crypt(req, &sg, NULL, head_data_len);
342	if (crypto_ahash_update(req))
343		return 1;
344
345	for (i = 0; i < shi->nr_frags; ++i) {
346		const skb_frag_t *f = &shi->frags[i];
347		unsigned int offset = skb_frag_off(f);
348		struct page *page;
349
350		page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
351		sg_set_page(&sg, page, skb_frag_size(f), offset_in_page(offset));
352		ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
353		if (crypto_ahash_update(req))
354			return 1;
355	}
356
357	skb_walk_frags(skb, frag_iter)
358		if (tcp_sigpool_hash_skb_data(hp, frag_iter, 0))
359			return 1;
360
361	return 0;
362}
363EXPORT_SYMBOL(tcp_sigpool_hash_skb_data);
364
365MODULE_LICENSE("GPL");
366MODULE_DESCRIPTION("Per-CPU pool of crypto requests");