1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * MIPS accelerated ChaCha and XChaCha stream ciphers,
  4 * including ChaCha20 (RFC7539)
  5 *
  6 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
  7 */
  8
  9#include <asm/byteorder.h>
 10#include <crypto/algapi.h>
 11#include <crypto/internal/chacha.h>
 12#include <crypto/internal/skcipher.h>
 13#include <linux/kernel.h>
 14#include <linux/module.h>
 15
 16asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src,
 17				  unsigned int bytes, int nrounds);
 18EXPORT_SYMBOL(chacha_crypt_arch);
 19
 20asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds);
 21EXPORT_SYMBOL(hchacha_block_arch);
 22
 23void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
 24{
 25	chacha_init_generic(state, key, iv);
 26}
 27EXPORT_SYMBOL(chacha_init_arch);
 28
 29static int chacha_mips_stream_xor(struct skcipher_request *req,
 30				  const struct chacha_ctx *ctx, const u8 *iv)
 31{
 32	struct skcipher_walk walk;
 33	u32 state[16];
 34	int err;
 35
 36	err = skcipher_walk_virt(&walk, req, false);
 37
 38	chacha_init_generic(state, ctx->key, iv);
 39
 40	while (walk.nbytes > 0) {
 41		unsigned int nbytes = walk.nbytes;
 42
 43		if (nbytes < walk.total)
 44			nbytes = round_down(nbytes, walk.stride);
 45
 46		chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr,
 47			     nbytes, ctx->nrounds);
 48		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 49	}
 50
 51	return err;
 52}
 53
 54static int chacha_mips(struct skcipher_request *req)
 55{
 56	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 57	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 58
 59	return chacha_mips_stream_xor(req, ctx, req->iv);
 60}
 61
 62static int xchacha_mips(struct skcipher_request *req)
 63{
 64	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 65	struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 66	struct chacha_ctx subctx;
 67	u32 state[16];
 68	u8 real_iv[16];
 69
 70	chacha_init_generic(state, ctx->key, req->iv);
 71
 72	hchacha_block(state, subctx.key, ctx->nrounds);
 73	subctx.nrounds = ctx->nrounds;
 74
 75	memcpy(&real_iv[0], req->iv + 24, 8);
 76	memcpy(&real_iv[8], req->iv + 16, 8);
 77	return chacha_mips_stream_xor(req, &subctx, real_iv);
 78}
 79
 80static struct skcipher_alg algs[] = {
 81	{
 82		.base.cra_name		= "chacha20",
 83		.base.cra_driver_name	= "chacha20-mips",
 84		.base.cra_priority	= 200,
 85		.base.cra_blocksize	= 1,
 86		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
 87		.base.cra_module	= THIS_MODULE,
 88
 89		.min_keysize		= CHACHA_KEY_SIZE,
 90		.max_keysize		= CHACHA_KEY_SIZE,
 91		.ivsize			= CHACHA_IV_SIZE,
 92		.chunksize		= CHACHA_BLOCK_SIZE,
 93		.setkey			= chacha20_setkey,
 94		.encrypt		= chacha_mips,
 95		.decrypt		= chacha_mips,
 96	}, {
 97		.base.cra_name		= "xchacha20",
 98		.base.cra_driver_name	= "xchacha20-mips",
 99		.base.cra_priority	= 200,
100		.base.cra_blocksize	= 1,
101		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
102		.base.cra_module	= THIS_MODULE,
103
104		.min_keysize		= CHACHA_KEY_SIZE,
105		.max_keysize		= CHACHA_KEY_SIZE,
106		.ivsize			= XCHACHA_IV_SIZE,
107		.chunksize		= CHACHA_BLOCK_SIZE,
108		.setkey			= chacha20_setkey,
109		.encrypt		= xchacha_mips,
110		.decrypt		= xchacha_mips,
111	}, {
112		.base.cra_name		= "xchacha12",
113		.base.cra_driver_name	= "xchacha12-mips",
114		.base.cra_priority	= 200,
115		.base.cra_blocksize	= 1,
116		.base.cra_ctxsize	= sizeof(struct chacha_ctx),
117		.base.cra_module	= THIS_MODULE,
118
119		.min_keysize		= CHACHA_KEY_SIZE,
120		.max_keysize		= CHACHA_KEY_SIZE,
121		.ivsize			= XCHACHA_IV_SIZE,
122		.chunksize		= CHACHA_BLOCK_SIZE,
123		.setkey			= chacha12_setkey,
124		.encrypt		= xchacha_mips,
125		.decrypt		= xchacha_mips,
126	}
127};
128
129static int __init chacha_simd_mod_init(void)
130{
131	return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ?
132		crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0;
133}
134
135static void __exit chacha_simd_mod_fini(void)
136{
137	if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER))
138		crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
139}
140
141module_init(chacha_simd_mod_init);
142module_exit(chacha_simd_mod_fini);
143
144MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)");
145MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
146MODULE_LICENSE("GPL v2");
147MODULE_ALIAS_CRYPTO("chacha20");
148MODULE_ALIAS_CRYPTO("chacha20-mips");
149MODULE_ALIAS_CRYPTO("xchacha20");
150MODULE_ALIAS_CRYPTO("xchacha20-mips");
151MODULE_ALIAS_CRYPTO("xchacha12");
152MODULE_ALIAS_CRYPTO("xchacha12-mips");