1/*
  2 * Cryptographic API.
  3 *
  4 * Glue code for the SHA512 Secure Hash Algorithm assembler
  5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
  6 *
  7 * This file is based on sha512_generic.c
  8 *
  9 * Copyright (C) 2013 Intel Corporation
 10 * Author: Tim Chen <tim.c.chen@linux.intel.com>
 11 *
 12 * This program is free software; you can redistribute it and/or modify it
 13 * under the terms of the GNU General Public License as published by the Free
 14 * Software Foundation; either version 2 of the License, or (at your option)
 15 * any later version.
 16 *
 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 24 * SOFTWARE.
 25 *
 26 */
 27
 28#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 29
 30#include <crypto/internal/hash.h>
 31#include <linux/init.h>
 32#include <linux/module.h>
 33#include <linux/mm.h>
 34#include <linux/cryptohash.h>
 35#include <linux/types.h>
 36#include <crypto/sha.h>
 37#include <crypto/sha512_base.h>
 38#include <asm/fpu/api.h>
 39
 40#include <linux/string.h>
 41
 42asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data,
 43				       u64 rounds);
 44
 45typedef void (sha512_transform_fn)(u64 *digest, const char *data, u64 rounds);
 46
 47static int sha512_update(struct shash_desc *desc, const u8 *data,
 48		       unsigned int len, sha512_transform_fn *sha512_xform)
 49{
 50	struct sha512_state *sctx = shash_desc_ctx(desc);
 51
 52	if (!irq_fpu_usable() ||
 53	    (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
 54		return crypto_sha512_update(desc, data, len);
 55
 56	/* make sure casting to sha512_block_fn() is safe */
 57	BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
 58
 59	kernel_fpu_begin();
 60	sha512_base_do_update(desc, data, len,
 61			      (sha512_block_fn *)sha512_xform);
 62	kernel_fpu_end();
 63
 64	return 0;
 65}
 66
 67static int sha512_finup(struct shash_desc *desc, const u8 *data,
 68	      unsigned int len, u8 *out, sha512_transform_fn *sha512_xform)
 69{
 70	if (!irq_fpu_usable())
 71		return crypto_sha512_finup(desc, data, len, out);
 72
 73	kernel_fpu_begin();
 74	if (len)
 75		sha512_base_do_update(desc, data, len,
 76				      (sha512_block_fn *)sha512_xform);
 77	sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_xform);
 78	kernel_fpu_end();
 79
 80	return sha512_base_finish(desc, out);
 81}
 82
 83static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
 84		       unsigned int len)
 85{
 86	return sha512_update(desc, data, len, sha512_transform_ssse3);
 87}
 88
 89static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
 90	      unsigned int len, u8 *out)
 91{
 92	return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
 93}
 94
 95/* Add padding and return the message digest. */
 96static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
 97{
 98	return sha512_ssse3_finup(desc, NULL, 0, out);
 99}
100
101static struct shash_alg sha512_ssse3_algs[] = { {
102	.digestsize	=	SHA512_DIGEST_SIZE,
103	.init		=	sha512_base_init,
104	.update		=	sha512_ssse3_update,
105	.final		=	sha512_ssse3_final,
106	.finup		=	sha512_ssse3_finup,
107	.descsize	=	sizeof(struct sha512_state),
108	.base		=	{
109		.cra_name	=	"sha512",
110		.cra_driver_name =	"sha512-ssse3",
111		.cra_priority	=	150,
112		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
113		.cra_blocksize	=	SHA512_BLOCK_SIZE,
114		.cra_module	=	THIS_MODULE,
115	}
116},  {
117	.digestsize	=	SHA384_DIGEST_SIZE,
118	.init		=	sha384_base_init,
119	.update		=	sha512_ssse3_update,
120	.final		=	sha512_ssse3_final,
121	.finup		=	sha512_ssse3_finup,
122	.descsize	=	sizeof(struct sha512_state),
123	.base		=	{
124		.cra_name	=	"sha384",
125		.cra_driver_name =	"sha384-ssse3",
126		.cra_priority	=	150,
127		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
128		.cra_blocksize	=	SHA384_BLOCK_SIZE,
129		.cra_module	=	THIS_MODULE,
130	}
131} };
132
133static int register_sha512_ssse3(void)
134{
135	if (boot_cpu_has(X86_FEATURE_SSSE3))
136		return crypto_register_shashes(sha512_ssse3_algs,
137			ARRAY_SIZE(sha512_ssse3_algs));
138	return 0;
139}
140
141static void unregister_sha512_ssse3(void)
142{
143	if (boot_cpu_has(X86_FEATURE_SSSE3))
144		crypto_unregister_shashes(sha512_ssse3_algs,
145			ARRAY_SIZE(sha512_ssse3_algs));
146}
147
148#ifdef CONFIG_AS_AVX
149asmlinkage void sha512_transform_avx(u64 *digest, const char *data,
150				     u64 rounds);
151static bool avx_usable(void)
152{
153	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
154		if (boot_cpu_has(X86_FEATURE_AVX))
155			pr_info("AVX detected but unusable.\n");
156		return false;
157	}
158
159	return true;
160}
161
162static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
163		       unsigned int len)
164{
165	return sha512_update(desc, data, len, sha512_transform_avx);
166}
167
168static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
169	      unsigned int len, u8 *out)
170{
171	return sha512_finup(desc, data, len, out, sha512_transform_avx);
172}
173
174/* Add padding and return the message digest. */
175static int sha512_avx_final(struct shash_desc *desc, u8 *out)
176{
177	return sha512_avx_finup(desc, NULL, 0, out);
178}
179
180static struct shash_alg sha512_avx_algs[] = { {
181	.digestsize	=	SHA512_DIGEST_SIZE,
182	.init		=	sha512_base_init,
183	.update		=	sha512_avx_update,
184	.final		=	sha512_avx_final,
185	.finup		=	sha512_avx_finup,
186	.descsize	=	sizeof(struct sha512_state),
187	.base		=	{
188		.cra_name	=	"sha512",
189		.cra_driver_name =	"sha512-avx",
190		.cra_priority	=	160,
191		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
192		.cra_blocksize	=	SHA512_BLOCK_SIZE,
193		.cra_module	=	THIS_MODULE,
194	}
195},  {
196	.digestsize	=	SHA384_DIGEST_SIZE,
197	.init		=	sha384_base_init,
198	.update		=	sha512_avx_update,
199	.final		=	sha512_avx_final,
200	.finup		=	sha512_avx_finup,
201	.descsize	=	sizeof(struct sha512_state),
202	.base		=	{
203		.cra_name	=	"sha384",
204		.cra_driver_name =	"sha384-avx",
205		.cra_priority	=	160,
206		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
207		.cra_blocksize	=	SHA384_BLOCK_SIZE,
208		.cra_module	=	THIS_MODULE,
209	}
210} };
211
212static int register_sha512_avx(void)
213{
214	if (avx_usable())
215		return crypto_register_shashes(sha512_avx_algs,
216			ARRAY_SIZE(sha512_avx_algs));
217	return 0;
218}
219
220static void unregister_sha512_avx(void)
221{
222	if (avx_usable())
223		crypto_unregister_shashes(sha512_avx_algs,
224			ARRAY_SIZE(sha512_avx_algs));
225}
226#else
227static inline int register_sha512_avx(void) { return 0; }
228static inline void unregister_sha512_avx(void) { }
229#endif
230
231#if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
232asmlinkage void sha512_transform_rorx(u64 *digest, const char *data,
233				      u64 rounds);
234
235static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
236		       unsigned int len)
237{
238	return sha512_update(desc, data, len, sha512_transform_rorx);
239}
240
241static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
242	      unsigned int len, u8 *out)
243{
244	return sha512_finup(desc, data, len, out, sha512_transform_rorx);
245}
246
247/* Add padding and return the message digest. */
248static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
249{
250	return sha512_avx2_finup(desc, NULL, 0, out);
251}
252
253static struct shash_alg sha512_avx2_algs[] = { {
254	.digestsize	=	SHA512_DIGEST_SIZE,
255	.init		=	sha512_base_init,
256	.update		=	sha512_avx2_update,
257	.final		=	sha512_avx2_final,
258	.finup		=	sha512_avx2_finup,
259	.descsize	=	sizeof(struct sha512_state),
260	.base		=	{
261		.cra_name	=	"sha512",
262		.cra_driver_name =	"sha512-avx2",
263		.cra_priority	=	170,
264		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
265		.cra_blocksize	=	SHA512_BLOCK_SIZE,
266		.cra_module	=	THIS_MODULE,
267	}
268},  {
269	.digestsize	=	SHA384_DIGEST_SIZE,
270	.init		=	sha384_base_init,
271	.update		=	sha512_avx2_update,
272	.final		=	sha512_avx2_final,
273	.finup		=	sha512_avx2_finup,
274	.descsize	=	sizeof(struct sha512_state),
275	.base		=	{
276		.cra_name	=	"sha384",
277		.cra_driver_name =	"sha384-avx2",
278		.cra_priority	=	170,
279		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
280		.cra_blocksize	=	SHA384_BLOCK_SIZE,
281		.cra_module	=	THIS_MODULE,
282	}
283} };
284
285static bool avx2_usable(void)
286{
287	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
288		    boot_cpu_has(X86_FEATURE_BMI2))
289		return true;
290
291	return false;
292}
293
294static int register_sha512_avx2(void)
295{
296	if (avx2_usable())
297		return crypto_register_shashes(sha512_avx2_algs,
298			ARRAY_SIZE(sha512_avx2_algs));
299	return 0;
300}
301
302static void unregister_sha512_avx2(void)
303{
304	if (avx2_usable())
305		crypto_unregister_shashes(sha512_avx2_algs,
306			ARRAY_SIZE(sha512_avx2_algs));
307}
308#else
309static inline int register_sha512_avx2(void) { return 0; }
310static inline void unregister_sha512_avx2(void) { }
311#endif
312
313static int __init sha512_ssse3_mod_init(void)
314{
315
316	if (register_sha512_ssse3())
317		goto fail;
318
319	if (register_sha512_avx()) {
320		unregister_sha512_ssse3();
321		goto fail;
322	}
323
324	if (register_sha512_avx2()) {
325		unregister_sha512_avx();
326		unregister_sha512_ssse3();
327		goto fail;
328	}
329
330	return 0;
331fail:
332	return -ENODEV;
333}
334
335static void __exit sha512_ssse3_mod_fini(void)
336{
337	unregister_sha512_avx2();
338	unregister_sha512_avx();
339	unregister_sha512_ssse3();
340}
341
342module_init(sha512_ssse3_mod_init);
343module_exit(sha512_ssse3_mod_fini);
344
345MODULE_LICENSE("GPL");
346MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
347
348MODULE_ALIAS_CRYPTO("sha512");
349MODULE_ALIAS_CRYPTO("sha512-ssse3");
350MODULE_ALIAS_CRYPTO("sha512-avx");
351MODULE_ALIAS_CRYPTO("sha512-avx2");
352MODULE_ALIAS_CRYPTO("sha384");
353MODULE_ALIAS_CRYPTO("sha384-ssse3");
354MODULE_ALIAS_CRYPTO("sha384-avx");
355MODULE_ALIAS_CRYPTO("sha384-avx2");