1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * af_alg: User-space algorithm interface
   4 *
   5 * This file provides the user-space API for algorithms.
   6 *
   7 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
 
 
 
 
 
 
   8 */
   9
  10#include <linux/atomic.h>
  11#include <crypto/if_alg.h>
  12#include <linux/crypto.h>
  13#include <linux/init.h>
  14#include <linux/kernel.h>
  15#include <linux/list.h>
  16#include <linux/module.h>
  17#include <linux/net.h>
  18#include <linux/rwsem.h>
  19#include <linux/sched.h>
  20#include <linux/sched/signal.h>
  21#include <linux/security.h>
  22
  23struct alg_type_list {
  24	const struct af_alg_type *type;
  25	struct list_head list;
  26};
  27
  28static atomic_long_t alg_memory_allocated;
  29
  30static struct proto alg_proto = {
  31	.name			= "ALG",
  32	.owner			= THIS_MODULE,
  33	.memory_allocated	= &alg_memory_allocated,
  34	.obj_size		= sizeof(struct alg_sock),
  35};
  36
  37static LIST_HEAD(alg_types);
  38static DECLARE_RWSEM(alg_types_sem);
  39
  40static const struct af_alg_type *alg_get_type(const char *name)
  41{
  42	const struct af_alg_type *type = ERR_PTR(-ENOENT);
  43	struct alg_type_list *node;
  44
  45	down_read(&alg_types_sem);
  46	list_for_each_entry(node, &alg_types, list) {
  47		if (strcmp(node->type->name, name))
  48			continue;
  49
  50		if (try_module_get(node->type->owner))
  51			type = node->type;
  52		break;
  53	}
  54	up_read(&alg_types_sem);
  55
  56	return type;
  57}
  58
  59int af_alg_register_type(const struct af_alg_type *type)
  60{
  61	struct alg_type_list *node;
  62	int err = -EEXIST;
  63
  64	down_write(&alg_types_sem);
  65	list_for_each_entry(node, &alg_types, list) {
  66		if (!strcmp(node->type->name, type->name))
  67			goto unlock;
  68	}
  69
  70	node = kmalloc(sizeof(*node), GFP_KERNEL);
  71	err = -ENOMEM;
  72	if (!node)
  73		goto unlock;
  74
  75	type->ops->owner = THIS_MODULE;
  76	if (type->ops_nokey)
  77		type->ops_nokey->owner = THIS_MODULE;
  78	node->type = type;
  79	list_add(&node->list, &alg_types);
  80	err = 0;
  81
  82unlock:
  83	up_write(&alg_types_sem);
  84
  85	return err;
  86}
  87EXPORT_SYMBOL_GPL(af_alg_register_type);
  88
  89int af_alg_unregister_type(const struct af_alg_type *type)
  90{
  91	struct alg_type_list *node;
  92	int err = -ENOENT;
  93
  94	down_write(&alg_types_sem);
  95	list_for_each_entry(node, &alg_types, list) {
  96		if (strcmp(node->type->name, type->name))
  97			continue;
  98
  99		list_del(&node->list);
 100		kfree(node);
 101		err = 0;
 102		break;
 103	}
 104	up_write(&alg_types_sem);
 105
 106	return err;
 107}
 108EXPORT_SYMBOL_GPL(af_alg_unregister_type);
 109
 110static void alg_do_release(const struct af_alg_type *type, void *private)
 111{
 112	if (!type)
 113		return;
 114
 115	type->release(private);
 116	module_put(type->owner);
 117}
 118
 119int af_alg_release(struct socket *sock)
 120{
 121	if (sock->sk) {
 122		sock_put(sock->sk);
 123		sock->sk = NULL;
 124	}
 125	return 0;
 126}
 127EXPORT_SYMBOL_GPL(af_alg_release);
 128
 129void af_alg_release_parent(struct sock *sk)
 130{
 131	struct alg_sock *ask = alg_sk(sk);
 132	unsigned int nokey = atomic_read(&ask->nokey_refcnt);
 133
 134	sk = ask->parent;
 135	ask = alg_sk(sk);
 136
 137	if (nokey)
 138		atomic_dec(&ask->nokey_refcnt);
 139
 140	if (atomic_dec_and_test(&ask->refcnt))
 141		sock_put(sk);
 142}
 143EXPORT_SYMBOL_GPL(af_alg_release_parent);
 144
 145static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
 146{
 147	const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY;
 148	struct sock *sk = sock->sk;
 149	struct alg_sock *ask = alg_sk(sk);
 150	struct sockaddr_alg *sa = (void *)uaddr;
 151	const struct af_alg_type *type;
 152	void *private;
 153	int err;
 154
 155	if (sock->state == SS_CONNECTED)
 156		return -EINVAL;
 157
 158	if (addr_len < sizeof(*sa))
 159		return -EINVAL;
 160
 161	/* If caller uses non-allowed flag, return error. */
 162	if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed))
 163		return -EINVAL;
 164
 165	sa->salg_type[sizeof(sa->salg_type) - 1] = 0;
 166	sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0;
 167
 168	type = alg_get_type(sa->salg_type);
 169	if (PTR_ERR(type) == -ENOENT) {
 170		request_module("algif-%s", sa->salg_type);
 171		type = alg_get_type(sa->salg_type);
 172	}
 173
 174	if (IS_ERR(type))
 175		return PTR_ERR(type);
 176
 177	private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask);
 178	if (IS_ERR(private)) {
 179		module_put(type->owner);
 180		return PTR_ERR(private);
 181	}
 182
 183	err = -EBUSY;
 184	lock_sock(sk);
 185	if (atomic_read(&ask->refcnt))
 186		goto unlock;
 187
 188	swap(ask->type, type);
 189	swap(ask->private, private);
 190
 191	err = 0;
 192
 193unlock:
 194	release_sock(sk);
 195
 196	alg_do_release(type, private);
 197
 198	return err;
 199}
 200
 201static int alg_setkey(struct sock *sk, sockptr_t ukey, unsigned int keylen)
 
 202{
 203	struct alg_sock *ask = alg_sk(sk);
 204	const struct af_alg_type *type = ask->type;
 205	u8 *key;
 206	int err;
 207
 208	key = sock_kmalloc(sk, keylen, GFP_KERNEL);
 209	if (!key)
 210		return -ENOMEM;
 211
 212	err = -EFAULT;
 213	if (copy_from_sockptr(key, ukey, keylen))
 214		goto out;
 215
 216	err = type->setkey(ask->private, key, keylen);
 217
 218out:
 219	sock_kzfree_s(sk, key, keylen);
 220
 221	return err;
 222}
 223
 224static int alg_setsockopt(struct socket *sock, int level, int optname,
 225			  sockptr_t optval, unsigned int optlen)
 226{
 227	struct sock *sk = sock->sk;
 228	struct alg_sock *ask = alg_sk(sk);
 229	const struct af_alg_type *type;
 230	int err = -EBUSY;
 231
 232	lock_sock(sk);
 233	if (atomic_read(&ask->refcnt) != atomic_read(&ask->nokey_refcnt))
 234		goto unlock;
 235
 236	type = ask->type;
 237
 238	err = -ENOPROTOOPT;
 239	if (level != SOL_ALG || !type)
 240		goto unlock;
 241
 242	switch (optname) {
 243	case ALG_SET_KEY:
 244		if (sock->state == SS_CONNECTED)
 245			goto unlock;
 246		if (!type->setkey)
 247			goto unlock;
 248
 249		err = alg_setkey(sk, optval, optlen);
 250		break;
 251	case ALG_SET_AEAD_AUTHSIZE:
 252		if (sock->state == SS_CONNECTED)
 253			goto unlock;
 254		if (!type->setauthsize)
 255			goto unlock;
 256		err = type->setauthsize(ask->private, optlen);
 257	}
 258
 259unlock:
 260	release_sock(sk);
 261
 262	return err;
 263}
 264
 265int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern)
 266{
 267	struct alg_sock *ask = alg_sk(sk);
 268	const struct af_alg_type *type;
 269	struct sock *sk2;
 270	unsigned int nokey;
 271	int err;
 272
 273	lock_sock(sk);
 274	type = ask->type;
 275
 276	err = -EINVAL;
 277	if (!type)
 278		goto unlock;
 279
 280	sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern);
 281	err = -ENOMEM;
 282	if (!sk2)
 283		goto unlock;
 284
 285	sock_init_data(newsock, sk2);
 286	security_sock_graft(sk2, newsock);
 287	security_sk_clone(sk, sk2);
 288
 289	err = type->accept(ask->private, sk2);
 290
 291	nokey = err == -ENOKEY;
 292	if (nokey && type->accept_nokey)
 293		err = type->accept_nokey(ask->private, sk2);
 294
 295	if (err)
 296		goto unlock;
 297
 298	if (atomic_inc_return_relaxed(&ask->refcnt) == 1)
 299		sock_hold(sk);
 300	if (nokey) {
 301		atomic_inc(&ask->nokey_refcnt);
 302		atomic_set(&alg_sk(sk2)->nokey_refcnt, 1);
 303	}
 
 
 
 
 304	alg_sk(sk2)->parent = sk;
 305	alg_sk(sk2)->type = type;
 306
 307	newsock->ops = type->ops;
 308	newsock->state = SS_CONNECTED;
 309
 310	if (nokey)
 311		newsock->ops = type->ops_nokey;
 312
 313	err = 0;
 314
 315unlock:
 316	release_sock(sk);
 317
 318	return err;
 319}
 320EXPORT_SYMBOL_GPL(af_alg_accept);
 321
 322static int alg_accept(struct socket *sock, struct socket *newsock, int flags,
 323		      bool kern)
 324{
 325	return af_alg_accept(sock->sk, newsock, kern);
 326}
 327
 328static const struct proto_ops alg_proto_ops = {
 329	.family		=	PF_ALG,
 330	.owner		=	THIS_MODULE,
 331
 332	.connect	=	sock_no_connect,
 333	.socketpair	=	sock_no_socketpair,
 334	.getname	=	sock_no_getname,
 335	.ioctl		=	sock_no_ioctl,
 336	.listen		=	sock_no_listen,
 337	.shutdown	=	sock_no_shutdown,
 
 338	.mmap		=	sock_no_mmap,
 339	.sendpage	=	sock_no_sendpage,
 340	.sendmsg	=	sock_no_sendmsg,
 341	.recvmsg	=	sock_no_recvmsg,
 
 342
 343	.bind		=	alg_bind,
 344	.release	=	af_alg_release,
 345	.setsockopt	=	alg_setsockopt,
 346	.accept		=	alg_accept,
 347};
 348
 349static void alg_sock_destruct(struct sock *sk)
 350{
 351	struct alg_sock *ask = alg_sk(sk);
 352
 353	alg_do_release(ask->type, ask->private);
 354}
 355
 356static int alg_create(struct net *net, struct socket *sock, int protocol,
 357		      int kern)
 358{
 359	struct sock *sk;
 360	int err;
 361
 362	if (sock->type != SOCK_SEQPACKET)
 363		return -ESOCKTNOSUPPORT;
 364	if (protocol != 0)
 365		return -EPROTONOSUPPORT;
 366
 367	err = -ENOMEM;
 368	sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern);
 369	if (!sk)
 370		goto out;
 371
 372	sock->ops = &alg_proto_ops;
 373	sock_init_data(sock, sk);
 374
 
 375	sk->sk_destruct = alg_sock_destruct;
 376
 377	return 0;
 378out:
 379	return err;
 380}
 381
 382static const struct net_proto_family alg_family = {
 383	.family	=	PF_ALG,
 384	.create	=	alg_create,
 385	.owner	=	THIS_MODULE,
 386};
 387
 388int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len)
 
 389{
 390	size_t off;
 391	ssize_t n;
 392	int npages, i;
 393
 394	n = iov_iter_get_pages(iter, sgl->pages, len, ALG_MAX_PAGES, &off);
 395	if (n < 0)
 396		return n;
 
 
 397
 398	npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 
 
 
 
 
 
 
 
 
 399	if (WARN_ON(npages == 0))
 400		return -EINVAL;
 401	/* Add one extra for linking */
 402	sg_init_table(sgl->sg, npages + 1);
 403
 404	for (i = 0, len = n; i < npages; i++) {
 
 
 405		int plen = min_t(int, len, PAGE_SIZE - off);
 406
 407		sg_set_page(sgl->sg + i, sgl->pages[i], plen, off);
 408
 409		off = 0;
 410		len -= plen;
 
 411	}
 412	sg_mark_end(sgl->sg + npages - 1);
 413	sgl->npages = npages;
 414
 415	return n;
 
 416}
 417EXPORT_SYMBOL_GPL(af_alg_make_sg);
 418
 419static void af_alg_link_sg(struct af_alg_sgl *sgl_prev,
 420			   struct af_alg_sgl *sgl_new)
 421{
 422	sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
 423	sg_chain(sgl_prev->sg, sgl_prev->npages + 1, sgl_new->sg);
 424}
 425
 426void af_alg_free_sg(struct af_alg_sgl *sgl)
 427{
 428	int i;
 429
 430	for (i = 0; i < sgl->npages; i++)
 
 431		put_page(sgl->pages[i]);
 
 432}
 433EXPORT_SYMBOL_GPL(af_alg_free_sg);
 434
 435static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
 436{
 437	struct cmsghdr *cmsg;
 438
 439	for_each_cmsghdr(cmsg, msg) {
 440		if (!CMSG_OK(msg, cmsg))
 441			return -EINVAL;
 442		if (cmsg->cmsg_level != SOL_ALG)
 443			continue;
 444
 445		switch (cmsg->cmsg_type) {
 446		case ALG_SET_IV:
 447			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
 448				return -EINVAL;
 449			con->iv = (void *)CMSG_DATA(cmsg);
 450			if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen +
 451						      sizeof(*con->iv)))
 452				return -EINVAL;
 453			break;
 454
 455		case ALG_SET_OP:
 456			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
 457				return -EINVAL;
 458			con->op = *(u32 *)CMSG_DATA(cmsg);
 459			break;
 460
 461		case ALG_SET_AEAD_ASSOCLEN:
 462			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
 463				return -EINVAL;
 464			con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg);
 465			break;
 466
 467		default:
 468			return -EINVAL;
 469		}
 470	}
 471
 472	return 0;
 473}
 
 474
 475/**
 476 * af_alg_alloc_tsgl - allocate the TX SGL
 477 *
 478 * @sk socket of connection to user space
 479 * @return: 0 upon success, < 0 upon error
 480 */
 481static int af_alg_alloc_tsgl(struct sock *sk)
 482{
 483	struct alg_sock *ask = alg_sk(sk);
 484	struct af_alg_ctx *ctx = ask->private;
 485	struct af_alg_tsgl *sgl;
 486	struct scatterlist *sg = NULL;
 487
 488	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
 489	if (!list_empty(&ctx->tsgl_list))
 490		sg = sgl->sg;
 491
 492	if (!sg || sgl->cur >= MAX_SGL_ENTS) {
 493		sgl = sock_kmalloc(sk,
 494				   struct_size(sgl, sg, (MAX_SGL_ENTS + 1)),
 495				   GFP_KERNEL);
 496		if (!sgl)
 497			return -ENOMEM;
 498
 499		sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
 500		sgl->cur = 0;
 501
 502		if (sg)
 503			sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
 504
 505		list_add_tail(&sgl->list, &ctx->tsgl_list);
 506	}
 507
 508	return 0;
 509}
 510
 511/**
 512 * aead_count_tsgl - Count number of TX SG entries
 513 *
 514 * The counting starts from the beginning of the SGL to @bytes. If
 515 * an offset is provided, the counting of the SG entries starts at the offset.
 516 *
 517 * @sk socket of connection to user space
 518 * @bytes Count the number of SG entries holding given number of bytes.
 519 * @offset Start the counting of SG entries from the given offset.
 520 * @return Number of TX SG entries found given the constraints
 521 */
 522unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset)
 523{
 524	const struct alg_sock *ask = alg_sk(sk);
 525	const struct af_alg_ctx *ctx = ask->private;
 526	const struct af_alg_tsgl *sgl;
 527	unsigned int i;
 528	unsigned int sgl_count = 0;
 529
 530	if (!bytes)
 531		return 0;
 532
 533	list_for_each_entry(sgl, &ctx->tsgl_list, list) {
 534		const struct scatterlist *sg = sgl->sg;
 535
 536		for (i = 0; i < sgl->cur; i++) {
 537			size_t bytes_count;
 538
 539			/* Skip offset */
 540			if (offset >= sg[i].length) {
 541				offset -= sg[i].length;
 542				bytes -= sg[i].length;
 543				continue;
 544			}
 545
 546			bytes_count = sg[i].length - offset;
 547
 548			offset = 0;
 549			sgl_count++;
 550
 551			/* If we have seen requested number of bytes, stop */
 552			if (bytes_count >= bytes)
 553				return sgl_count;
 554
 555			bytes -= bytes_count;
 556		}
 557	}
 558
 559	return sgl_count;
 560}
 561EXPORT_SYMBOL_GPL(af_alg_count_tsgl);
 562
 563/**
 564 * aead_pull_tsgl - Release the specified buffers from TX SGL
 565 *
 566 * If @dst is non-null, reassign the pages to dst. The caller must release
 567 * the pages. If @dst_offset is given only reassign the pages to @dst starting
 568 * at the @dst_offset (byte). The caller must ensure that @dst is large
 569 * enough (e.g. by using af_alg_count_tsgl with the same offset).
 570 *
 571 * @sk socket of connection to user space
 572 * @used Number of bytes to pull from TX SGL
 573 * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The
 574 *	caller must release the buffers in dst.
 575 * @dst_offset Reassign the TX SGL from given offset. All buffers before
 576 *	       reaching the offset is released.
 577 */
 578void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst,
 579		      size_t dst_offset)
 580{
 581	struct alg_sock *ask = alg_sk(sk);
 582	struct af_alg_ctx *ctx = ask->private;
 583	struct af_alg_tsgl *sgl;
 584	struct scatterlist *sg;
 585	unsigned int i, j = 0;
 586
 587	while (!list_empty(&ctx->tsgl_list)) {
 588		sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl,
 589				       list);
 590		sg = sgl->sg;
 591
 592		for (i = 0; i < sgl->cur; i++) {
 593			size_t plen = min_t(size_t, used, sg[i].length);
 594			struct page *page = sg_page(sg + i);
 595
 596			if (!page)
 597				continue;
 598
 599			/*
 600			 * Assumption: caller created af_alg_count_tsgl(len)
 601			 * SG entries in dst.
 602			 */
 603			if (dst) {
 604				if (dst_offset >= plen) {
 605					/* discard page before offset */
 606					dst_offset -= plen;
 607				} else {
 608					/* reassign page to dst after offset */
 609					get_page(page);
 610					sg_set_page(dst + j, page,
 611						    plen - dst_offset,
 612						    sg[i].offset + dst_offset);
 613					dst_offset = 0;
 614					j++;
 615				}
 616			}
 617
 618			sg[i].length -= plen;
 619			sg[i].offset += plen;
 620
 621			used -= plen;
 622			ctx->used -= plen;
 623
 624			if (sg[i].length)
 625				return;
 626
 627			put_page(page);
 628			sg_assign_page(sg + i, NULL);
 629		}
 630
 631		list_del(&sgl->list);
 632		sock_kfree_s(sk, sgl, struct_size(sgl, sg, MAX_SGL_ENTS + 1));
 633	}
 634
 635	if (!ctx->used)
 636		ctx->merge = 0;
 637	ctx->init = ctx->more;
 638}
 639EXPORT_SYMBOL_GPL(af_alg_pull_tsgl);
 640
 641/**
 642 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request
 643 *
 644 * @areq Request holding the TX and RX SGL
 645 */
 646static void af_alg_free_areq_sgls(struct af_alg_async_req *areq)
 647{
 648	struct sock *sk = areq->sk;
 649	struct alg_sock *ask = alg_sk(sk);
 650	struct af_alg_ctx *ctx = ask->private;
 651	struct af_alg_rsgl *rsgl, *tmp;
 652	struct scatterlist *tsgl;
 653	struct scatterlist *sg;
 654	unsigned int i;
 655
 656	list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
 657		atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused);
 658		af_alg_free_sg(&rsgl->sgl);
 659		list_del(&rsgl->list);
 660		if (rsgl != &areq->first_rsgl)
 661			sock_kfree_s(sk, rsgl, sizeof(*rsgl));
 662	}
 663
 664	tsgl = areq->tsgl;
 665	if (tsgl) {
 666		for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
 667			if (!sg_page(sg))
 668				continue;
 669			put_page(sg_page(sg));
 670		}
 671
 672		sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
 673	}
 674}
 675
 676/**
 677 * af_alg_wait_for_wmem - wait for availability of writable memory
 678 *
 679 * @sk socket of connection to user space
 680 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
 681 * @return 0 when writable memory is available, < 0 upon error
 682 */
 683static int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags)
 684{
 685	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 686	int err = -ERESTARTSYS;
 687	long timeout;
 688
 689	if (flags & MSG_DONTWAIT)
 690		return -EAGAIN;
 691
 692	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 693
 694	add_wait_queue(sk_sleep(sk), &wait);
 695	for (;;) {
 696		if (signal_pending(current))
 697			break;
 698		timeout = MAX_SCHEDULE_TIMEOUT;
 699		if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) {
 700			err = 0;
 701			break;
 702		}
 703	}
 704	remove_wait_queue(sk_sleep(sk), &wait);
 705
 706	return err;
 707}
 708
 709/**
 710 * af_alg_wmem_wakeup - wakeup caller when writable memory is available
 711 *
 712 * @sk socket of connection to user space
 713 */
 714void af_alg_wmem_wakeup(struct sock *sk)
 715{
 716	struct socket_wq *wq;
 717
 718	if (!af_alg_writable(sk))
 719		return;
 720
 721	rcu_read_lock();
 722	wq = rcu_dereference(sk->sk_wq);
 723	if (skwq_has_sleeper(wq))
 724		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
 725							   EPOLLRDNORM |
 726							   EPOLLRDBAND);
 727	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
 728	rcu_read_unlock();
 729}
 730EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
 731
 732/**
 733 * af_alg_wait_for_data - wait for availability of TX data
 734 *
 735 * @sk socket of connection to user space
 736 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
 737 * @min Set to minimum request size if partial requests are allowed.
 738 * @return 0 when writable memory is available, < 0 upon error
 739 */
 740int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min)
 741{
 742	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 743	struct alg_sock *ask = alg_sk(sk);
 744	struct af_alg_ctx *ctx = ask->private;
 745	long timeout;
 746	int err = -ERESTARTSYS;
 747
 748	if (flags & MSG_DONTWAIT)
 749		return -EAGAIN;
 750
 751	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 752
 753	add_wait_queue(sk_sleep(sk), &wait);
 754	for (;;) {
 755		if (signal_pending(current))
 756			break;
 757		timeout = MAX_SCHEDULE_TIMEOUT;
 758		if (sk_wait_event(sk, &timeout,
 759				  ctx->init && (!ctx->more ||
 760						(min && ctx->used >= min)),
 761				  &wait)) {
 762			err = 0;
 763			break;
 764		}
 765	}
 766	remove_wait_queue(sk_sleep(sk), &wait);
 767
 768	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 769
 770	return err;
 771}
 772EXPORT_SYMBOL_GPL(af_alg_wait_for_data);
 773
 774/**
 775 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel
 776 *
 777 * @sk socket of connection to user space
 778 */
 779static void af_alg_data_wakeup(struct sock *sk)
 780{
 781	struct alg_sock *ask = alg_sk(sk);
 782	struct af_alg_ctx *ctx = ask->private;
 783	struct socket_wq *wq;
 784
 785	if (!ctx->used)
 786		return;
 787
 788	rcu_read_lock();
 789	wq = rcu_dereference(sk->sk_wq);
 790	if (skwq_has_sleeper(wq))
 791		wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
 792							   EPOLLRDNORM |
 793							   EPOLLRDBAND);
 794	sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
 795	rcu_read_unlock();
 796}
 797
 798/**
 799 * af_alg_sendmsg - implementation of sendmsg system call handler
 800 *
 801 * The sendmsg system call handler obtains the user data and stores it
 802 * in ctx->tsgl_list. This implies allocation of the required numbers of
 803 * struct af_alg_tsgl.
 804 *
 805 * In addition, the ctx is filled with the information sent via CMSG.
 806 *
 807 * @sock socket of connection to user space
 808 * @msg message from user space
 809 * @size size of message from user space
 810 * @ivsize the size of the IV for the cipher operation to verify that the
 811 *	   user-space-provided IV has the right size
 812 * @return the number of copied data upon success, < 0 upon error
 813 */
 814int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
 815		   unsigned int ivsize)
 816{
 817	struct sock *sk = sock->sk;
 818	struct alg_sock *ask = alg_sk(sk);
 819	struct af_alg_ctx *ctx = ask->private;
 820	struct af_alg_tsgl *sgl;
 821	struct af_alg_control con = {};
 822	long copied = 0;
 823	bool enc = false;
 824	bool init = false;
 825	int err = 0;
 826
 827	if (msg->msg_controllen) {
 828		err = af_alg_cmsg_send(msg, &con);
 829		if (err)
 830			return err;
 831
 832		init = true;
 833		switch (con.op) {
 834		case ALG_OP_ENCRYPT:
 835			enc = true;
 836			break;
 837		case ALG_OP_DECRYPT:
 838			enc = false;
 839			break;
 840		default:
 841			return -EINVAL;
 842		}
 843
 844		if (con.iv && con.iv->ivlen != ivsize)
 845			return -EINVAL;
 846	}
 847
 848	lock_sock(sk);
 849	if (ctx->init && !ctx->more) {
 850		if (ctx->used) {
 851			err = -EINVAL;
 852			goto unlock;
 853		}
 854
 855		pr_info_once(
 856			"%s sent an empty control message without MSG_MORE.\n",
 857			current->comm);
 858	}
 859	ctx->init = true;
 860
 861	if (init) {
 862		ctx->enc = enc;
 863		if (con.iv)
 864			memcpy(ctx->iv, con.iv->iv, ivsize);
 865
 866		ctx->aead_assoclen = con.aead_assoclen;
 867	}
 868
 869	while (size) {
 870		struct scatterlist *sg;
 871		size_t len = size;
 872		size_t plen;
 873
 874		/* use the existing memory in an allocated page */
 875		if (ctx->merge) {
 876			sgl = list_entry(ctx->tsgl_list.prev,
 877					 struct af_alg_tsgl, list);
 878			sg = sgl->sg + sgl->cur - 1;
 879			len = min_t(size_t, len,
 880				    PAGE_SIZE - sg->offset - sg->length);
 881
 882			err = memcpy_from_msg(page_address(sg_page(sg)) +
 883					      sg->offset + sg->length,
 884					      msg, len);
 885			if (err)
 886				goto unlock;
 887
 888			sg->length += len;
 889			ctx->merge = (sg->offset + sg->length) &
 890				     (PAGE_SIZE - 1);
 891
 892			ctx->used += len;
 893			copied += len;
 894			size -= len;
 895			continue;
 896		}
 897
 898		if (!af_alg_writable(sk)) {
 899			err = af_alg_wait_for_wmem(sk, msg->msg_flags);
 900			if (err)
 901				goto unlock;
 902		}
 903
 904		/* allocate a new page */
 905		len = min_t(unsigned long, len, af_alg_sndbuf(sk));
 906
 907		err = af_alg_alloc_tsgl(sk);
 908		if (err)
 909			goto unlock;
 910
 911		sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl,
 912				 list);
 913		sg = sgl->sg;
 914		if (sgl->cur)
 915			sg_unmark_end(sg + sgl->cur - 1);
 916
 917		do {
 918			unsigned int i = sgl->cur;
 919
 920			plen = min_t(size_t, len, PAGE_SIZE);
 921
 922			sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
 923			if (!sg_page(sg + i)) {
 924				err = -ENOMEM;
 925				goto unlock;
 926			}
 927
 928			err = memcpy_from_msg(page_address(sg_page(sg + i)),
 929					      msg, plen);
 930			if (err) {
 931				__free_page(sg_page(sg + i));
 932				sg_assign_page(sg + i, NULL);
 933				goto unlock;
 934			}
 935
 936			sg[i].length = plen;
 937			len -= plen;
 938			ctx->used += plen;
 939			copied += plen;
 940			size -= plen;
 941			sgl->cur++;
 942		} while (len && sgl->cur < MAX_SGL_ENTS);
 943
 944		if (!size)
 945			sg_mark_end(sg + sgl->cur - 1);
 946
 947		ctx->merge = plen & (PAGE_SIZE - 1);
 948	}
 949
 950	err = 0;
 951
 952	ctx->more = msg->msg_flags & MSG_MORE;
 953
 954unlock:
 955	af_alg_data_wakeup(sk);
 956	release_sock(sk);
 957
 958	return copied ?: err;
 959}
 960EXPORT_SYMBOL_GPL(af_alg_sendmsg);
 961
 962/**
 963 * af_alg_sendpage - sendpage system call handler
 964 *
 965 * This is a generic implementation of sendpage to fill ctx->tsgl_list.
 966 */
 967ssize_t af_alg_sendpage(struct socket *sock, struct page *page,
 968			int offset, size_t size, int flags)
 969{
 970	struct sock *sk = sock->sk;
 971	struct alg_sock *ask = alg_sk(sk);
 972	struct af_alg_ctx *ctx = ask->private;
 973	struct af_alg_tsgl *sgl;
 974	int err = -EINVAL;
 975
 976	if (flags & MSG_SENDPAGE_NOTLAST)
 977		flags |= MSG_MORE;
 978
 979	lock_sock(sk);
 980	if (!ctx->more && ctx->used)
 981		goto unlock;
 982
 983	if (!size)
 984		goto done;
 985
 986	if (!af_alg_writable(sk)) {
 987		err = af_alg_wait_for_wmem(sk, flags);
 988		if (err)
 989			goto unlock;
 990	}
 991
 992	err = af_alg_alloc_tsgl(sk);
 993	if (err)
 994		goto unlock;
 995
 996	ctx->merge = 0;
 997	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
 998
 999	if (sgl->cur)
1000		sg_unmark_end(sgl->sg + sgl->cur - 1);
1001
1002	sg_mark_end(sgl->sg + sgl->cur);
1003
1004	get_page(page);
1005	sg_set_page(sgl->sg + sgl->cur, page, size, offset);
1006	sgl->cur++;
1007	ctx->used += size;
1008
1009done:
1010	ctx->more = flags & MSG_MORE;
1011
1012unlock:
1013	af_alg_data_wakeup(sk);
1014	release_sock(sk);
1015
1016	return err ?: size;
1017}
1018EXPORT_SYMBOL_GPL(af_alg_sendpage);
1019
1020/**
1021 * af_alg_free_resources - release resources required for crypto request
1022 */
1023void af_alg_free_resources(struct af_alg_async_req *areq)
1024{
1025	struct sock *sk = areq->sk;
1026
1027	af_alg_free_areq_sgls(areq);
1028	sock_kfree_s(sk, areq, areq->areqlen);
1029}
1030EXPORT_SYMBOL_GPL(af_alg_free_resources);
1031
1032/**
1033 * af_alg_async_cb - AIO callback handler
1034 *
1035 * This handler cleans up the struct af_alg_async_req upon completion of the
1036 * AIO operation.
1037 *
1038 * The number of bytes to be generated with the AIO operation must be set
1039 * in areq->outlen before the AIO callback handler is invoked.
1040 */
1041void af_alg_async_cb(struct crypto_async_request *_req, int err)
1042{
1043	struct af_alg_async_req *areq = _req->data;
1044	struct sock *sk = areq->sk;
1045	struct kiocb *iocb = areq->iocb;
1046	unsigned int resultlen;
1047
1048	/* Buffer size written by crypto operation. */
1049	resultlen = areq->outlen;
1050
1051	af_alg_free_resources(areq);
1052	sock_put(sk);
1053
1054	iocb->ki_complete(iocb, err ? err : (int)resultlen, 0);
1055}
1056EXPORT_SYMBOL_GPL(af_alg_async_cb);
1057
1058/**
1059 * af_alg_poll - poll system call handler
1060 */
1061__poll_t af_alg_poll(struct file *file, struct socket *sock,
1062			 poll_table *wait)
1063{
1064	struct sock *sk = sock->sk;
1065	struct alg_sock *ask = alg_sk(sk);
1066	struct af_alg_ctx *ctx = ask->private;
1067	__poll_t mask;
1068
1069	sock_poll_wait(file, sock, wait);
1070	mask = 0;
1071
1072	if (!ctx->more || ctx->used)
1073		mask |= EPOLLIN | EPOLLRDNORM;
1074
1075	if (af_alg_writable(sk))
1076		mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
1077
1078	return mask;
1079}
1080EXPORT_SYMBOL_GPL(af_alg_poll);
1081
1082/**
1083 * af_alg_alloc_areq - allocate struct af_alg_async_req
1084 *
1085 * @sk socket of connection to user space
1086 * @areqlen size of struct af_alg_async_req + crypto_*_reqsize
1087 * @return allocated data structure or ERR_PTR upon error
1088 */
1089struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
1090					   unsigned int areqlen)
1091{
1092	struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
1093
1094	if (unlikely(!areq))
1095		return ERR_PTR(-ENOMEM);
1096
1097	areq->areqlen = areqlen;
1098	areq->sk = sk;
1099	areq->last_rsgl = NULL;
1100	INIT_LIST_HEAD(&areq->rsgl_list);
1101	areq->tsgl = NULL;
1102	areq->tsgl_entries = 0;
1103
1104	return areq;
1105}
1106EXPORT_SYMBOL_GPL(af_alg_alloc_areq);
1107
1108/**
1109 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto
1110 *		     operation
1111 *
1112 * @sk socket of connection to user space
1113 * @msg user space message
1114 * @flags flags used to invoke recvmsg with
1115 * @areq instance of the cryptographic request that will hold the RX SGL
1116 * @maxsize maximum number of bytes to be pulled from user space
1117 * @outlen number of bytes in the RX SGL
1118 * @return 0 on success, < 0 upon error
1119 */
1120int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags,
1121		    struct af_alg_async_req *areq, size_t maxsize,
1122		    size_t *outlen)
1123{
1124	struct alg_sock *ask = alg_sk(sk);
1125	struct af_alg_ctx *ctx = ask->private;
1126	size_t len = 0;
1127
1128	while (maxsize > len && msg_data_left(msg)) {
1129		struct af_alg_rsgl *rsgl;
1130		size_t seglen;
1131		int err;
1132
1133		/* limit the amount of readable buffers */
1134		if (!af_alg_readable(sk))
1135			break;
1136
1137		seglen = min_t(size_t, (maxsize - len),
1138			       msg_data_left(msg));
1139
1140		if (list_empty(&areq->rsgl_list)) {
1141			rsgl = &areq->first_rsgl;
1142		} else {
1143			rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
1144			if (unlikely(!rsgl))
1145				return -ENOMEM;
1146		}
1147
1148		rsgl->sgl.npages = 0;
1149		list_add_tail(&rsgl->list, &areq->rsgl_list);
1150
1151		/* make one iovec available as scatterlist */
1152		err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
1153		if (err < 0) {
1154			rsgl->sg_num_bytes = 0;
1155			return err;
1156		}
1157
1158		/* chain the new scatterlist with previous one */
1159		if (areq->last_rsgl)
1160			af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl);
1161
1162		areq->last_rsgl = rsgl;
1163		len += err;
1164		atomic_add(err, &ctx->rcvused);
1165		rsgl->sg_num_bytes = err;
1166		iov_iter_advance(&msg->msg_iter, err);
1167	}
1168
1169	*outlen = len;
1170	return 0;
1171}
1172EXPORT_SYMBOL_GPL(af_alg_get_rsgl);
1173
1174static int __init af_alg_init(void)
1175{
1176	int err = proto_register(&alg_proto, 0);
1177
1178	if (err)
1179		goto out;
1180
1181	err = sock_register(&alg_family);
1182	if (err != 0)
1183		goto out_unregister_proto;
1184
1185out:
1186	return err;
1187
1188out_unregister_proto:
1189	proto_unregister(&alg_proto);
1190	goto out;
1191}
1192
1193static void __exit af_alg_exit(void)
1194{
1195	sock_unregister(PF_ALG);
1196	proto_unregister(&alg_proto);
1197}
1198
1199module_init(af_alg_init);
1200module_exit(af_alg_exit);
1201MODULE_LICENSE("GPL");
1202MODULE_ALIAS_NETPROTO(AF_ALG);