1/*
   2 * af_alg: User-space algorithm interface
   3 *
   4 * This file provides the user-space API for algorithms.
   5 *
   6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
   7 *
   8 * This program is free software; you can redistribute it and/or modify it
   9 * under the terms of the GNU General Public License as published by the Free
  10 * Software Foundation; either version 2 of the License, or (at your option)
  11 * any later version.
  12 *
  13 */
  14
  15#include <linux/atomic.h>
  16#include <crypto/if_alg.h>
  17#include <linux/crypto.h>
  18#include <linux/init.h>
  19#include <linux/kernel.h>
  20#include <linux/list.h>
  21#include <linux/module.h>
  22#include <linux/net.h>
  23#include <linux/rwsem.h>
  24#include <linux/sched/signal.h>
  25#include <linux/security.h>
  26
  27struct alg_type_list {
  28	const struct af_alg_type *type;
  29	struct list_head list;
  30};
  31
  32static atomic_long_t alg_memory_allocated;
  33
  34static struct proto alg_proto = {
  35	.name			= "ALG",
  36	.owner			= THIS_MODULE,
  37	.memory_allocated	= &alg_memory_allocated,
  38	.obj_size		= sizeof(struct alg_sock),
  39};
  40
  41static LIST_HEAD(alg_types);
  42static DECLARE_RWSEM(alg_types_sem);
  43
  44static const struct af_alg_type *alg_get_type(const char *name)
  45{
  46	const struct af_alg_type *type = ERR_PTR(-ENOENT);
  47	struct alg_type_list *node;
  48
  49	down_read(&alg_types_sem);
  50	list_for_each_entry(node, &alg_types, list) {
  51		if (strcmp(node->type->name, name))
  52			continue;
  53
  54		if (try_module_get(node->type->owner))
  55			type = node->type;
  56		break;
  57	}
  58	up_read(&alg_types_sem);
  59
  60	return type;
  61}
  62
  63int af_alg_register_type(const struct af_alg_type *type)
  64{
  65	struct alg_type_list *node;
  66	int err = -EEXIST;
  67
  68	down_write(&alg_types_sem);
  69	list_for_each_entry(node, &alg_types, list) {
  70		if (!strcmp(node->type->name, type->name))
  71			goto unlock;
  72	}
  73
  74	node = kmalloc(sizeof(*node), GFP_KERNEL);
  75	err = -ENOMEM;
  76	if (!node)
  77		goto unlock;
  78
  79	type->ops->owner = THIS_MODULE;
  80	if (type->ops_nokey)
  81		type->ops_nokey->owner = THIS_MODULE;
  82	node->type = type;
  83	list_add(&node->list, &alg_types);
  84	err = 0;
  85
  86unlock:
  87	up_write(&alg_types_sem);
  88
  89	return err;
  90}
  91EXPORT_SYMBOL_GPL(af_alg_register_type);
  92
  93int af_alg_unregister_type(const struct af_alg_type *type)
  94{
  95	struct alg_type_list *node;
  96	int err = -ENOENT;
  97
  98	down_write(&alg_types_sem);
  99	list_for_each_entry(node, &alg_types, list) {
 100		if (strcmp(node->type->name, type->name))
 101			continue;
 102
 103		list_del(&node->list);
 104		kfree(node);
 105		err = 0;
 106		break;
 107	}
 108	up_write(&alg_types_sem);
 109
 110	return err;
 111}
 112EXPORT_SYMBOL_GPL(af_alg_unregister_type);
 113
 114static void alg_do_release(const struct af_alg_type *type, void *private)
 115{
 116	if (!type)
 117		return;
 118
 119	type->release(private);
 120	module_put(type->owner);
 121}
 122
 123int af_alg_release(struct socket *sock)
 124{
 125	if (sock->sk)
 126		sock_put(sock->sk);
 127	return 0;
 128}
 129EXPORT_SYMBOL_GPL(af_alg_release);
 130
 131void af_alg_release_parent(struct sock *sk)
 132{
 133	struct alg_sock *ask = alg_sk(sk);
 134	unsigned int nokey = ask->nokey_refcnt;
 135	bool last = nokey && !ask->refcnt;
 136
 137	sk = ask->parent;
 138	ask = alg_sk(sk);
 139
 140	lock_sock(sk);
 141	ask->nokey_refcnt -= nokey;
 142	if (!last)
 143		last = !--ask->refcnt;
 144	release_sock(sk);
 145
 146	if (last)
 147		sock_put(sk);
 148}
 149EXPORT_SYMBOL_GPL(af_alg_release_parent);
 150
 151static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
 152{
 153	const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY;
 154	struct sock *sk = sock->sk;
 155	struct alg_sock *ask = alg_sk(sk);
 156	struct sockaddr_alg *sa = (void *)uaddr;
 157	const struct af_alg_type *type;
 158	void *private;
 159	int err;
 160
 161	if (sock->state == SS_CONNECTED)
 162		return -EINVAL;
 163
 164	if (addr_len < sizeof(*sa))
 165		return -EINVAL;
 166
 167	/* If caller uses non-allowed flag, return error. */
 168	if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed))
 169		return -EINVAL;
 170
 171	sa->salg_type[sizeof(sa->salg_type) - 1] = 0;
 172	sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0;
 173
 174	type = alg_get_type(sa->salg_type);
 175	if (IS_ERR(type) && PTR_ERR(type) == -ENOENT) {
 176		request_module("algif-%s", sa->salg_type);
 177		type = alg_get_type(sa->salg_type);
 178	}
 179
 180	if (IS_ERR(type))
 181		return PTR_ERR(type);
 182
 183	private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask);
 184	if (IS_ERR(private)) {
 185		module_put(type->owner);
 186		return PTR_ERR(private);
 187	}
 188
 189	err = -EBUSY;
 190	lock_sock(sk);
 191	if (ask->refcnt | ask->nokey_refcnt)
 192		goto unlock;
 193
 194	swap(ask->type, type);
 195	swap(ask->private, private);
 196
 197	err = 0;
 198
 199unlock:
 200	release_sock(sk);
 201
 202	alg_do_release(type, private);
 203
 204	return err;
 205}
 206
 207static int alg_setkey(struct sock *sk, char __user *ukey,
 208		      unsigned int keylen)
 209{
 210	struct alg_sock *ask = alg_sk(sk);
 211	const struct af_alg_type *type = ask->type;
 212	u8 *key;
 213	int err;
 214
 215	key = sock_kmalloc(sk, keylen, GFP_KERNEL);
 216	if (!key)
 217		return -ENOMEM;
 218
 219	err = -EFAULT;
 220	if (copy_from_user(key, ukey, keylen))
 221		goto out;
 222
 223	err = type->setkey(ask->private, key, keylen);
 224
 225out:
 226	sock_kzfree_s(sk, key, keylen);
 227
 228	return err;
 229}
 230
 231static int alg_setsockopt(struct socket *sock, int level, int optname,
 232			  char __user *optval, unsigned int optlen)
 233{
 234	struct sock *sk = sock->sk;
 235	struct alg_sock *ask = alg_sk(sk);
 236	const struct af_alg_type *type;
 237	int err = -EBUSY;
 238
 239	lock_sock(sk);
 240	if (ask->refcnt)
 241		goto unlock;
 242
 243	type = ask->type;
 244
 245	err = -ENOPROTOOPT;
 246	if (level != SOL_ALG || !type)
 247		goto unlock;
 248
 249	switch (optname) {
 250	case ALG_SET_KEY:
 251		if (sock->state == SS_CONNECTED)
 252			goto unlock;
 253		if (!type->setkey)
 254			goto unlock;
 255
 256		err = alg_setkey(sk, optval, optlen);
 257		break;
 258	case ALG_SET_AEAD_AUTHSIZE:
 259		if (sock->state == SS_CONNECTED)
 260			goto unlock;
 261		if (!type->setauthsize)
 262			goto unlock;
 263		err = type->setauthsize(ask->private, optlen);
 264	}
 265
 266unlock:
 267	release_sock(sk);
 268
 269	return err;
 270}
 271
 272int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern)
 273{
 274	struct alg_sock *ask = alg_sk(sk);
 275	const struct af_alg_type *type;
 276	struct sock *sk2;
 277	unsigned int nokey;
 278	int err;
 279
 280	lock_sock(sk);
 281	type = ask->type;
 282
 283	err = -EINVAL;
 284	if (!type)
 285		goto unlock;
 286
 287	sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern);
 288	err = -ENOMEM;
 289	if (!sk2)
 290		goto unlock;
 291
 292	sock_init_data(newsock, sk2);
 293	security_sock_graft(sk2, newsock);
 294	security_sk_clone(sk, sk2);
 295
 296	err = type->accept(ask->private, sk2);
 297
 298	nokey = err == -ENOKEY;
 299	if (nokey && type->accept_nokey)
 300		err = type->accept_nokey(ask->private, sk2);
 301
 302	if (err)
 303		goto unlock;
 
 304
 305	sk2->sk_family = PF_ALG;
 306
 307	if (nokey || !ask->refcnt++)
 308		sock_hold(sk);
 309	ask->nokey_refcnt += nokey;
 310	alg_sk(sk2)->parent = sk;
 311	alg_sk(sk2)->type = type;
 312	alg_sk(sk2)->nokey_refcnt = nokey;
 313
 314	newsock->ops = type->ops;
 315	newsock->state = SS_CONNECTED;
 316
 317	if (nokey)
 318		newsock->ops = type->ops_nokey;
 319
 320	err = 0;
 321
 322unlock:
 323	release_sock(sk);
 324
 325	return err;
 326}
 327EXPORT_SYMBOL_GPL(af_alg_accept);
 328
 329static int alg_accept(struct socket *sock, struct socket *newsock, int flags,
 330		      bool kern)
 331{
 332	return af_alg_accept(sock->sk, newsock, kern);
 333}
 334
 335static const struct proto_ops alg_proto_ops = {
 336	.family		=	PF_ALG,
 337	.owner		=	THIS_MODULE,
 338
 339	.connect	=	sock_no_connect,
 340	.socketpair	=	sock_no_socketpair,
 341	.getname	=	sock_no_getname,
 342	.ioctl		=	sock_no_ioctl,
 343	.listen		=	sock_no_listen,
 344	.shutdown	=	sock_no_shutdown,
 345	.getsockopt	=	sock_no_getsockopt,
 346	.mmap		=	sock_no_mmap,
 347	.sendpage	=	sock_no_sendpage,
 348	.sendmsg	=	sock_no_sendmsg,
 349	.recvmsg	=	sock_no_recvmsg,
 350	.poll		=	sock_no_poll,
 351
 352	.bind		=	alg_bind,
 353	.release	=	af_alg_release,
 354	.setsockopt	=	alg_setsockopt,
 355	.accept		=	alg_accept,
 356};
 357
 358static void alg_sock_destruct(struct sock *sk)
 359{
 360	struct alg_sock *ask = alg_sk(sk);
 361
 362	alg_do_release(ask->type, ask->private);
 363}
 364
 365static int alg_create(struct net *net, struct socket *sock, int protocol,
 366		      int kern)
 367{
 368	struct sock *sk;
 369	int err;
 370
 371	if (sock->type != SOCK_SEQPACKET)
 372		return -ESOCKTNOSUPPORT;
 373	if (protocol != 0)
 374		return -EPROTONOSUPPORT;
 375
 376	err = -ENOMEM;
 377	sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern);
 378	if (!sk)
 379		goto out;
 380
 381	sock->ops = &alg_proto_ops;
 382	sock_init_data(sock, sk);
 383
 384	sk->sk_family = PF_ALG;
 385	sk->sk_destruct = alg_sock_destruct;
 386
 387	return 0;
 388out:
 389	return err;
 390}
 391
 392static const struct net_proto_family alg_family = {
 393	.family	=	PF_ALG,
 394	.create	=	alg_create,
 395	.owner	=	THIS_MODULE,
 396};
 397
 398int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len)
 
 399{
 400	size_t off;
 401	ssize_t n;
 402	int npages, i;
 403
 404	n = iov_iter_get_pages(iter, sgl->pages, len, ALG_MAX_PAGES, &off);
 405	if (n < 0)
 406		return n;
 
 
 
 
 
 
 
 
 
 
 
 407
 408	npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 409	if (WARN_ON(npages == 0))
 410		return -EINVAL;
 411	/* Add one extra for linking */
 412	sg_init_table(sgl->sg, npages + 1);
 413
 414	for (i = 0, len = n; i < npages; i++) {
 
 
 
 
 415		int plen = min_t(int, len, PAGE_SIZE - off);
 416
 417		sg_set_page(sgl->sg + i, sgl->pages[i], plen, off);
 418
 419		off = 0;
 420		len -= plen;
 
 421	}
 422	sg_mark_end(sgl->sg + npages - 1);
 423	sgl->npages = npages;
 424
 425	return n;
 
 426}
 427EXPORT_SYMBOL_GPL(af_alg_make_sg);
 428
 429void af_alg_link_sg(struct af_alg_sgl *sgl_prev, struct af_alg_sgl *sgl_new)
 430{
 431	sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
 432	sg_chain(sgl_prev->sg, sgl_prev->npages + 1, sgl_new->sg);
 433}
 434EXPORT_SYMBOL_GPL(af_alg_link_sg);
 435
 436void af_alg_free_sg(struct af_alg_sgl *sgl)
 437{
 438	int i;
 439
 440	for (i = 0; i < sgl->npages; i++)
 
 441		put_page(sgl->pages[i]);
 
 442}
 443EXPORT_SYMBOL_GPL(af_alg_free_sg);
 444
 445int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
 446{
 447	struct cmsghdr *cmsg;
 448
 449	for_each_cmsghdr(cmsg, msg) {
 450		if (!CMSG_OK(msg, cmsg))
 451			return -EINVAL;
 452		if (cmsg->cmsg_level != SOL_ALG)
 453			continue;
 454
 455		switch (cmsg->cmsg_type) {
 456		case ALG_SET_IV:
 457			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
 458				return -EINVAL;
 459			con->iv = (void *)CMSG_DATA(cmsg);
 460			if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen +
 461						      sizeof(*con->iv)))
 462				return -EINVAL;
 463			break;
 464
 465		case ALG_SET_OP:
 466			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
 467				return -EINVAL;
 468			con->op = *(u32 *)CMSG_DATA(cmsg);
 469			break;
 470
 471		case ALG_SET_AEAD_ASSOCLEN:
 472			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
 473				return -EINVAL;
 474			con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg);
 475			break;
 476
 477		default:
 478			return -EINVAL;
 479		}
 480	}
 481
 482	return 0;
 483}
 484EXPORT_SYMBOL_GPL(af_alg_cmsg_send);
 485
 486/**
 487 * af_alg_alloc_tsgl - allocate the TX SGL
 488 *
 489 * @sk socket of connection to user space
 490 * @return: 0 upon success, < 0 upon error
 491 */
 492int af_alg_alloc_tsgl(struct sock *sk)
 493{
 494	struct alg_sock *ask = alg_sk(sk);
 495	struct af_alg_ctx *ctx = ask->private;
 496	struct af_alg_tsgl *sgl;
 497	struct scatterlist *sg = NULL;
 498
 499	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
 500	if (!list_empty(&ctx->tsgl_list))
 501		sg = sgl->sg;
 502
 503	if (!sg || sgl->cur >= MAX_SGL_ENTS) {
 504		sgl = sock_kmalloc(sk, sizeof(*sgl) +
 505				       sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
 506				   GFP_KERNEL);
 507		if (!sgl)
 508			return -ENOMEM;
 509
 510		sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
 511		sgl->cur = 0;
 512
 513		if (sg)
 514			sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
 515
 516		list_add_tail(&sgl->list, &ctx->tsgl_list);
 517	}
 518
 519	return 0;
 520}
 521EXPORT_SYMBOL_GPL(af_alg_alloc_tsgl);
 522
 523/**
 524 * aead_count_tsgl - Count number of TX SG entries
 525 *
 526 * The counting starts from the beginning of the SGL to @bytes. If
 527 * an offset is provided, the counting of the SG entries starts at the offset.
 528 *
 529 * @sk socket of connection to user space
 530 * @bytes Count the number of SG entries holding given number of bytes.
 531 * @offset Start the counting of SG entries from the given offset.
 532 * @return Number of TX SG entries found given the constraints
 533 */
 534unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset)
 535{
 536	struct alg_sock *ask = alg_sk(sk);
 537	struct af_alg_ctx *ctx = ask->private;
 538	struct af_alg_tsgl *sgl, *tmp;
 539	unsigned int i;
 540	unsigned int sgl_count = 0;
 541
 542	if (!bytes)
 543		return 0;
 544
 545	list_for_each_entry_safe(sgl, tmp, &ctx->tsgl_list, list) {
 546		struct scatterlist *sg = sgl->sg;
 547
 548		for (i = 0; i < sgl->cur; i++) {
 549			size_t bytes_count;
 550
 551			/* Skip offset */
 552			if (offset >= sg[i].length) {
 553				offset -= sg[i].length;
 554				bytes -= sg[i].length;
 555				continue;
 556			}
 557
 558			bytes_count = sg[i].length - offset;
 559
 560			offset = 0;
 561			sgl_count++;
 562
 563			/* If we have seen requested number of bytes, stop */
 564			if (bytes_count >= bytes)
 565				return sgl_count;
 566
 567			bytes -= bytes_count;
 568		}
 569	}
 570
 571	return sgl_count;
 572}
 573EXPORT_SYMBOL_GPL(af_alg_count_tsgl);
 574
 575/**
 576 * aead_pull_tsgl - Release the specified buffers from TX SGL
 577 *
 578 * If @dst is non-null, reassign the pages to dst. The caller must release
 579 * the pages. If @dst_offset is given only reassign the pages to @dst starting
 580 * at the @dst_offset (byte). The caller must ensure that @dst is large
 581 * enough (e.g. by using af_alg_count_tsgl with the same offset).
 582 *
 583 * @sk socket of connection to user space
 584 * @used Number of bytes to pull from TX SGL
 585 * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The
 586 *	caller must release the buffers in dst.
 587 * @dst_offset Reassign the TX SGL from given offset. All buffers before
 588 *	       reaching the offset is released.
 589 */
 590void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst,
 591		      size_t dst_offset)
 592{
 593	struct alg_sock *ask = alg_sk(sk);
 594	struct af_alg_ctx *ctx = ask->private;
 595	struct af_alg_tsgl *sgl;
 596	struct scatterlist *sg;
 597	unsigned int i, j = 0;
 598
 599	while (!list_empty(&ctx->tsgl_list)) {
 600		sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl,
 601				       list);
 602		sg = sgl->sg;
 603
 604		for (i = 0; i < sgl->cur; i++) {
 605			size_t plen = min_t(size_t, used, sg[i].length);
 606			struct page *page = sg_page(sg + i);
 607
 608			if (!page)
 609				continue;
 610
 611			/*
 612			 * Assumption: caller created af_alg_count_tsgl(len)
 613			 * SG entries in dst.
 614			 */
 615			if (dst) {
 616				if (dst_offset >= plen) {
 617					/* discard page before offset */
 618					dst_offset -= plen;
 619				} else {
 620					/* reassign page to dst after offset */
 621					get_page(page);
 622					sg_set_page(dst + j, page,
 623						    plen - dst_offset,
 624						    sg[i].offset + dst_offset);
 625					dst_offset = 0;
 626					j++;
 627				}
 628			}
 629
 630			sg[i].length -= plen;
 631			sg[i].offset += plen;
 632
 633			used -= plen;
 634			ctx->used -= plen;
 635
 636			if (sg[i].length)
 637				return;
 638
 639			put_page(page);
 640			sg_assign_page(sg + i, NULL);
 641		}
 642
 643		list_del(&sgl->list);
 644		sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
 645						     (MAX_SGL_ENTS + 1));
 646	}
 647
 648	if (!ctx->used)
 649		ctx->merge = 0;
 650}
 651EXPORT_SYMBOL_GPL(af_alg_pull_tsgl);
 652
 653/**
 654 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request
 655 *
 656 * @areq Request holding the TX and RX SGL
 657 */
 658void af_alg_free_areq_sgls(struct af_alg_async_req *areq)
 659{
 660	struct sock *sk = areq->sk;
 661	struct alg_sock *ask = alg_sk(sk);
 662	struct af_alg_ctx *ctx = ask->private;
 663	struct af_alg_rsgl *rsgl, *tmp;
 664	struct scatterlist *tsgl;
 665	struct scatterlist *sg;
 666	unsigned int i;
 667
 668	list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
 669		atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused);
 670		af_alg_free_sg(&rsgl->sgl);
 671		list_del(&rsgl->list);
 672		if (rsgl != &areq->first_rsgl)
 673			sock_kfree_s(sk, rsgl, sizeof(*rsgl));
 674	}
 675
 676	tsgl = areq->tsgl;
 677	if (tsgl) {
 678		for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
 679			if (!sg_page(sg))
 680				continue;
 681			put_page(sg_page(sg));
 682		}
 683
 684		sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
 685	}
 686}
 687EXPORT_SYMBOL_GPL(af_alg_free_areq_sgls);
 688
 689/**
 690 * af_alg_wait_for_wmem - wait for availability of writable memory
 691 *
 692 * @sk socket of connection to user space
 693 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
 694 * @return 0 when writable memory is available, < 0 upon error
 695 */
 696int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags)
 697{
 698	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 699	int err = -ERESTARTSYS;
 700	long timeout;
 701
 702	if (flags & MSG_DONTWAIT)
 703		return -EAGAIN;
 704
 705	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 706
 707	add_wait_queue(sk_sleep(sk), &wait);
 708	for (;;) {
 709		if (signal_pending(current))
 710			break;
 711		timeout = MAX_SCHEDULE_TIMEOUT;
 712		if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) {
 713			err = 0;
 714			break;
 715		}
 716	}
 717	remove_wait_queue(sk_sleep(sk), &wait);
 718
 719	return err;
 720}
 721EXPORT_SYMBOL_GPL(af_alg_wait_for_wmem);
 722
 723/**
 724 * af_alg_wmem_wakeup - wakeup caller when writable memory is available
 725 *
 726 * @sk socket of connection to user space
 727 */
 728void af_alg_wmem_wakeup(struct sock *sk)
 729{
 730	struct socket_wq *wq;
 731
 732	if (!af_alg_writable(sk))
 733		return;
 734
 735	rcu_read_lock();
 736	wq = rcu_dereference(sk->sk_wq);
 737	if (skwq_has_sleeper(wq))
 738		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
 739							   EPOLLRDNORM |
 740							   EPOLLRDBAND);
 741	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
 742	rcu_read_unlock();
 743}
 744EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
 745
 746/**
 747 * af_alg_wait_for_data - wait for availability of TX data
 748 *
 749 * @sk socket of connection to user space
 750 * @flags If MSG_DONTWAIT is set, then only report if function would sleep
 751 * @return 0 when writable memory is available, < 0 upon error
 752 */
 753int af_alg_wait_for_data(struct sock *sk, unsigned flags)
 754{
 755	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 756	struct alg_sock *ask = alg_sk(sk);
 757	struct af_alg_ctx *ctx = ask->private;
 758	long timeout;
 759	int err = -ERESTARTSYS;
 760
 761	if (flags & MSG_DONTWAIT)
 762		return -EAGAIN;
 763
 764	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 765
 766	add_wait_queue(sk_sleep(sk), &wait);
 767	for (;;) {
 768		if (signal_pending(current))
 769			break;
 770		timeout = MAX_SCHEDULE_TIMEOUT;
 771		if (sk_wait_event(sk, &timeout, (ctx->used || !ctx->more),
 772				  &wait)) {
 773			err = 0;
 774			break;
 775		}
 776	}
 777	remove_wait_queue(sk_sleep(sk), &wait);
 778
 779	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 780
 781	return err;
 782}
 783EXPORT_SYMBOL_GPL(af_alg_wait_for_data);
 784
 785/**
 786 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel
 787 *
 788 * @sk socket of connection to user space
 789 */
 790
 791void af_alg_data_wakeup(struct sock *sk)
 792{
 793	struct alg_sock *ask = alg_sk(sk);
 794	struct af_alg_ctx *ctx = ask->private;
 795	struct socket_wq *wq;
 796
 797	if (!ctx->used)
 798		return;
 799
 800	rcu_read_lock();
 801	wq = rcu_dereference(sk->sk_wq);
 802	if (skwq_has_sleeper(wq))
 803		wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
 804							   EPOLLRDNORM |
 805							   EPOLLRDBAND);
 806	sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
 807	rcu_read_unlock();
 808}
 809EXPORT_SYMBOL_GPL(af_alg_data_wakeup);
 810
 811/**
 812 * af_alg_sendmsg - implementation of sendmsg system call handler
 813 *
 814 * The sendmsg system call handler obtains the user data and stores it
 815 * in ctx->tsgl_list. This implies allocation of the required numbers of
 816 * struct af_alg_tsgl.
 817 *
 818 * In addition, the ctx is filled with the information sent via CMSG.
 819 *
 820 * @sock socket of connection to user space
 821 * @msg message from user space
 822 * @size size of message from user space
 823 * @ivsize the size of the IV for the cipher operation to verify that the
 824 *	   user-space-provided IV has the right size
 825 * @return the number of copied data upon success, < 0 upon error
 826 */
 827int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
 828		   unsigned int ivsize)
 829{
 830	struct sock *sk = sock->sk;
 831	struct alg_sock *ask = alg_sk(sk);
 832	struct af_alg_ctx *ctx = ask->private;
 833	struct af_alg_tsgl *sgl;
 834	struct af_alg_control con = {};
 835	long copied = 0;
 836	bool enc = 0;
 837	bool init = 0;
 838	int err = 0;
 839
 840	if (msg->msg_controllen) {
 841		err = af_alg_cmsg_send(msg, &con);
 842		if (err)
 843			return err;
 844
 845		init = 1;
 846		switch (con.op) {
 847		case ALG_OP_ENCRYPT:
 848			enc = 1;
 849			break;
 850		case ALG_OP_DECRYPT:
 851			enc = 0;
 852			break;
 853		default:
 854			return -EINVAL;
 855		}
 856
 857		if (con.iv && con.iv->ivlen != ivsize)
 858			return -EINVAL;
 859	}
 860
 861	lock_sock(sk);
 862	if (!ctx->more && ctx->used) {
 863		err = -EINVAL;
 864		goto unlock;
 865	}
 866
 867	if (init) {
 868		ctx->enc = enc;
 869		if (con.iv)
 870			memcpy(ctx->iv, con.iv->iv, ivsize);
 871
 872		ctx->aead_assoclen = con.aead_assoclen;
 873	}
 874
 875	while (size) {
 876		struct scatterlist *sg;
 877		size_t len = size;
 878		size_t plen;
 879
 880		/* use the existing memory in an allocated page */
 881		if (ctx->merge) {
 882			sgl = list_entry(ctx->tsgl_list.prev,
 883					 struct af_alg_tsgl, list);
 884			sg = sgl->sg + sgl->cur - 1;
 885			len = min_t(size_t, len,
 886				    PAGE_SIZE - sg->offset - sg->length);
 887
 888			err = memcpy_from_msg(page_address(sg_page(sg)) +
 889					      sg->offset + sg->length,
 890					      msg, len);
 891			if (err)
 892				goto unlock;
 893
 894			sg->length += len;
 895			ctx->merge = (sg->offset + sg->length) &
 896				     (PAGE_SIZE - 1);
 897
 898			ctx->used += len;
 899			copied += len;
 900			size -= len;
 901			continue;
 902		}
 903
 904		if (!af_alg_writable(sk)) {
 905			err = af_alg_wait_for_wmem(sk, msg->msg_flags);
 906			if (err)
 907				goto unlock;
 908		}
 909
 910		/* allocate a new page */
 911		len = min_t(unsigned long, len, af_alg_sndbuf(sk));
 912
 913		err = af_alg_alloc_tsgl(sk);
 914		if (err)
 915			goto unlock;
 916
 917		sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl,
 918				 list);
 919		sg = sgl->sg;
 920		if (sgl->cur)
 921			sg_unmark_end(sg + sgl->cur - 1);
 922
 923		do {
 924			unsigned int i = sgl->cur;
 925
 926			plen = min_t(size_t, len, PAGE_SIZE);
 927
 928			sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
 929			if (!sg_page(sg + i)) {
 930				err = -ENOMEM;
 931				goto unlock;
 932			}
 933
 934			err = memcpy_from_msg(page_address(sg_page(sg + i)),
 935					      msg, plen);
 936			if (err) {
 937				__free_page(sg_page(sg + i));
 938				sg_assign_page(sg + i, NULL);
 939				goto unlock;
 940			}
 941
 942			sg[i].length = plen;
 943			len -= plen;
 944			ctx->used += plen;
 945			copied += plen;
 946			size -= plen;
 947			sgl->cur++;
 948		} while (len && sgl->cur < MAX_SGL_ENTS);
 949
 950		if (!size)
 951			sg_mark_end(sg + sgl->cur - 1);
 952
 953		ctx->merge = plen & (PAGE_SIZE - 1);
 954	}
 955
 956	err = 0;
 957
 958	ctx->more = msg->msg_flags & MSG_MORE;
 959
 960unlock:
 961	af_alg_data_wakeup(sk);
 962	release_sock(sk);
 963
 964	return copied ?: err;
 965}
 966EXPORT_SYMBOL_GPL(af_alg_sendmsg);
 967
 968/**
 969 * af_alg_sendpage - sendpage system call handler
 970 *
 971 * This is a generic implementation of sendpage to fill ctx->tsgl_list.
 972 */
 973ssize_t af_alg_sendpage(struct socket *sock, struct page *page,
 974			int offset, size_t size, int flags)
 975{
 976	struct sock *sk = sock->sk;
 977	struct alg_sock *ask = alg_sk(sk);
 978	struct af_alg_ctx *ctx = ask->private;
 979	struct af_alg_tsgl *sgl;
 980	int err = -EINVAL;
 981
 982	if (flags & MSG_SENDPAGE_NOTLAST)
 983		flags |= MSG_MORE;
 984
 985	lock_sock(sk);
 986	if (!ctx->more && ctx->used)
 987		goto unlock;
 988
 989	if (!size)
 990		goto done;
 991
 992	if (!af_alg_writable(sk)) {
 993		err = af_alg_wait_for_wmem(sk, flags);
 994		if (err)
 995			goto unlock;
 996	}
 997
 998	err = af_alg_alloc_tsgl(sk);
 999	if (err)
1000		goto unlock;
1001
1002	ctx->merge = 0;
1003	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
1004
1005	if (sgl->cur)
1006		sg_unmark_end(sgl->sg + sgl->cur - 1);
1007
1008	sg_mark_end(sgl->sg + sgl->cur);
1009
1010	get_page(page);
1011	sg_set_page(sgl->sg + sgl->cur, page, size, offset);
1012	sgl->cur++;
1013	ctx->used += size;
1014
1015done:
1016	ctx->more = flags & MSG_MORE;
1017
1018unlock:
1019	af_alg_data_wakeup(sk);
1020	release_sock(sk);
1021
1022	return err ?: size;
1023}
1024EXPORT_SYMBOL_GPL(af_alg_sendpage);
1025
1026/**
1027 * af_alg_free_resources - release resources required for crypto request
1028 */
1029void af_alg_free_resources(struct af_alg_async_req *areq)
1030{
1031	struct sock *sk = areq->sk;
1032
1033	af_alg_free_areq_sgls(areq);
1034	sock_kfree_s(sk, areq, areq->areqlen);
1035}
1036EXPORT_SYMBOL_GPL(af_alg_free_resources);
1037
1038/**
1039 * af_alg_async_cb - AIO callback handler
1040 *
1041 * This handler cleans up the struct af_alg_async_req upon completion of the
1042 * AIO operation.
1043 *
1044 * The number of bytes to be generated with the AIO operation must be set
1045 * in areq->outlen before the AIO callback handler is invoked.
1046 */
1047void af_alg_async_cb(struct crypto_async_request *_req, int err)
1048{
1049	struct af_alg_async_req *areq = _req->data;
1050	struct sock *sk = areq->sk;
1051	struct kiocb *iocb = areq->iocb;
1052	unsigned int resultlen;
1053
1054	/* Buffer size written by crypto operation. */
1055	resultlen = areq->outlen;
1056
1057	af_alg_free_resources(areq);
1058	sock_put(sk);
1059
1060	iocb->ki_complete(iocb, err ? err : resultlen, 0);
1061}
1062EXPORT_SYMBOL_GPL(af_alg_async_cb);
1063
1064/**
1065 * af_alg_poll - poll system call handler
1066 */
1067__poll_t af_alg_poll(struct file *file, struct socket *sock,
1068			 poll_table *wait)
1069{
1070	struct sock *sk = sock->sk;
1071	struct alg_sock *ask = alg_sk(sk);
1072	struct af_alg_ctx *ctx = ask->private;
1073	__poll_t mask;
1074
1075	sock_poll_wait(file, sk_sleep(sk), wait);
1076	mask = 0;
1077
1078	if (!ctx->more || ctx->used)
1079		mask |= EPOLLIN | EPOLLRDNORM;
1080
1081	if (af_alg_writable(sk))
1082		mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
1083
1084	return mask;
1085}
1086EXPORT_SYMBOL_GPL(af_alg_poll);
1087
1088/**
1089 * af_alg_alloc_areq - allocate struct af_alg_async_req
1090 *
1091 * @sk socket of connection to user space
1092 * @areqlen size of struct af_alg_async_req + crypto_*_reqsize
1093 * @return allocated data structure or ERR_PTR upon error
1094 */
1095struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
1096					   unsigned int areqlen)
1097{
1098	struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
1099
1100	if (unlikely(!areq))
1101		return ERR_PTR(-ENOMEM);
1102
1103	areq->areqlen = areqlen;
1104	areq->sk = sk;
1105	areq->last_rsgl = NULL;
1106	INIT_LIST_HEAD(&areq->rsgl_list);
1107	areq->tsgl = NULL;
1108	areq->tsgl_entries = 0;
1109
1110	return areq;
1111}
1112EXPORT_SYMBOL_GPL(af_alg_alloc_areq);
1113
1114/**
1115 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto
1116 *		     operation
1117 *
1118 * @sk socket of connection to user space
1119 * @msg user space message
1120 * @flags flags used to invoke recvmsg with
1121 * @areq instance of the cryptographic request that will hold the RX SGL
1122 * @maxsize maximum number of bytes to be pulled from user space
1123 * @outlen number of bytes in the RX SGL
1124 * @return 0 on success, < 0 upon error
1125 */
1126int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags,
1127		    struct af_alg_async_req *areq, size_t maxsize,
1128		    size_t *outlen)
1129{
1130	struct alg_sock *ask = alg_sk(sk);
1131	struct af_alg_ctx *ctx = ask->private;
1132	size_t len = 0;
1133
1134	while (maxsize > len && msg_data_left(msg)) {
1135		struct af_alg_rsgl *rsgl;
1136		size_t seglen;
1137		int err;
1138
1139		/* limit the amount of readable buffers */
1140		if (!af_alg_readable(sk))
1141			break;
1142
1143		seglen = min_t(size_t, (maxsize - len),
1144			       msg_data_left(msg));
1145
1146		if (list_empty(&areq->rsgl_list)) {
1147			rsgl = &areq->first_rsgl;
1148		} else {
1149			rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
1150			if (unlikely(!rsgl))
1151				return -ENOMEM;
1152		}
1153
1154		rsgl->sgl.npages = 0;
1155		list_add_tail(&rsgl->list, &areq->rsgl_list);
1156
1157		/* make one iovec available as scatterlist */
1158		err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
1159		if (err < 0)
1160			return err;
1161
1162		/* chain the new scatterlist with previous one */
1163		if (areq->last_rsgl)
1164			af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl);
1165
1166		areq->last_rsgl = rsgl;
1167		len += err;
1168		atomic_add(err, &ctx->rcvused);
1169		rsgl->sg_num_bytes = err;
1170		iov_iter_advance(&msg->msg_iter, err);
1171	}
1172
1173	*outlen = len;
1174	return 0;
1175}
1176EXPORT_SYMBOL_GPL(af_alg_get_rsgl);
1177
1178static int __init af_alg_init(void)
1179{
1180	int err = proto_register(&alg_proto, 0);
1181
1182	if (err)
1183		goto out;
1184
1185	err = sock_register(&alg_family);
1186	if (err != 0)
1187		goto out_unregister_proto;
1188
1189out:
1190	return err;
1191
1192out_unregister_proto:
1193	proto_unregister(&alg_proto);
1194	goto out;
1195}
1196
1197static void __exit af_alg_exit(void)
1198{
1199	sock_unregister(PF_ALG);
1200	proto_unregister(&alg_proto);
1201}
1202
1203module_init(af_alg_init);
1204module_exit(af_alg_exit);
1205MODULE_LICENSE("GPL");
1206MODULE_ALIAS_NETPROTO(AF_ALG);