Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
4 *
5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6 */
7
8#include <linux/module.h>
9#include <linux/types.h>
10#include <linux/crypto.h>
11#include <linux/err.h>
12#include <crypto/algapi.h>
13#include <crypto/internal/simd.h>
14#include <crypto/serpent.h>
15
16#include "serpent-avx.h"
17#include "ecb_cbc_helpers.h"
18
19#define SERPENT_AVX2_PARALLEL_BLOCKS 16
20
21/* 16-way AVX2 parallel cipher functions */
22asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
23asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
24asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
25
26static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
27 const u8 *key, unsigned int keylen)
28{
29 return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
30}
31
32static int ecb_encrypt(struct skcipher_request *req)
33{
34 ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
35 ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way);
36 ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
37 ECB_BLOCK(1, __serpent_encrypt);
38 ECB_WALK_END();
39}
40
41static int ecb_decrypt(struct skcipher_request *req)
42{
43 ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
44 ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way);
45 ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
46 ECB_BLOCK(1, __serpent_decrypt);
47 ECB_WALK_END();
48}
49
50static int cbc_encrypt(struct skcipher_request *req)
51{
52 CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
53 CBC_ENC_BLOCK(__serpent_encrypt);
54 CBC_WALK_END();
55}
56
57static int cbc_decrypt(struct skcipher_request *req)
58{
59 CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
60 CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way);
61 CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
62 CBC_DEC_BLOCK(1, __serpent_decrypt);
63 CBC_WALK_END();
64}
65
66static struct skcipher_alg serpent_algs[] = {
67 {
68 .base.cra_name = "__ecb(serpent)",
69 .base.cra_driver_name = "__ecb-serpent-avx2",
70 .base.cra_priority = 600,
71 .base.cra_flags = CRYPTO_ALG_INTERNAL,
72 .base.cra_blocksize = SERPENT_BLOCK_SIZE,
73 .base.cra_ctxsize = sizeof(struct serpent_ctx),
74 .base.cra_module = THIS_MODULE,
75 .min_keysize = SERPENT_MIN_KEY_SIZE,
76 .max_keysize = SERPENT_MAX_KEY_SIZE,
77 .setkey = serpent_setkey_skcipher,
78 .encrypt = ecb_encrypt,
79 .decrypt = ecb_decrypt,
80 }, {
81 .base.cra_name = "__cbc(serpent)",
82 .base.cra_driver_name = "__cbc-serpent-avx2",
83 .base.cra_priority = 600,
84 .base.cra_flags = CRYPTO_ALG_INTERNAL,
85 .base.cra_blocksize = SERPENT_BLOCK_SIZE,
86 .base.cra_ctxsize = sizeof(struct serpent_ctx),
87 .base.cra_module = THIS_MODULE,
88 .min_keysize = SERPENT_MIN_KEY_SIZE,
89 .max_keysize = SERPENT_MAX_KEY_SIZE,
90 .ivsize = SERPENT_BLOCK_SIZE,
91 .setkey = serpent_setkey_skcipher,
92 .encrypt = cbc_encrypt,
93 .decrypt = cbc_decrypt,
94 },
95};
96
97static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
98
99static int __init serpent_avx2_init(void)
100{
101 const char *feature_name;
102
103 if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
104 pr_info("AVX2 instructions are not detected.\n");
105 return -ENODEV;
106 }
107 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
108 &feature_name)) {
109 pr_info("CPU feature '%s' is not supported.\n", feature_name);
110 return -ENODEV;
111 }
112
113 return simd_register_skciphers_compat(serpent_algs,
114 ARRAY_SIZE(serpent_algs),
115 serpent_simd_algs);
116}
117
118static void __exit serpent_avx2_fini(void)
119{
120 simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
121 serpent_simd_algs);
122}
123
124module_init(serpent_avx2_init);
125module_exit(serpent_avx2_fini);
126
127MODULE_LICENSE("GPL");
128MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
129MODULE_ALIAS_CRYPTO("serpent");
130MODULE_ALIAS_CRYPTO("serpent-asm");
1/*
2 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
3 *
4 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 */
12
13#include <linux/module.h>
14#include <linux/types.h>
15#include <linux/crypto.h>
16#include <linux/err.h>
17#include <crypto/ablk_helper.h>
18#include <crypto/algapi.h>
19#include <crypto/ctr.h>
20#include <crypto/lrw.h>
21#include <crypto/xts.h>
22#include <crypto/serpent.h>
23#include <asm/fpu/api.h>
24#include <asm/crypto/serpent-avx.h>
25#include <asm/crypto/glue_helper.h>
26
27#define SERPENT_AVX2_PARALLEL_BLOCKS 16
28
29/* 16-way AVX2 parallel cipher functions */
30asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
31 const u8 *src);
32asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
33 const u8 *src);
34asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
35
36asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
37 le128 *iv);
38asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
39 const u8 *src, le128 *iv);
40asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
41 const u8 *src, le128 *iv);
42
43static const struct common_glue_ctx serpent_enc = {
44 .num_funcs = 3,
45 .fpu_blocks_limit = 8,
46
47 .funcs = { {
48 .num_blocks = 16,
49 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
50 }, {
51 .num_blocks = 8,
52 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
53 }, {
54 .num_blocks = 1,
55 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
56 } }
57};
58
59static const struct common_glue_ctx serpent_ctr = {
60 .num_funcs = 3,
61 .fpu_blocks_limit = 8,
62
63 .funcs = { {
64 .num_blocks = 16,
65 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
66 }, {
67 .num_blocks = 8,
68 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
69 }, {
70 .num_blocks = 1,
71 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
72 } }
73};
74
75static const struct common_glue_ctx serpent_enc_xts = {
76 .num_funcs = 3,
77 .fpu_blocks_limit = 8,
78
79 .funcs = { {
80 .num_blocks = 16,
81 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
82 }, {
83 .num_blocks = 8,
84 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
85 }, {
86 .num_blocks = 1,
87 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
88 } }
89};
90
91static const struct common_glue_ctx serpent_dec = {
92 .num_funcs = 3,
93 .fpu_blocks_limit = 8,
94
95 .funcs = { {
96 .num_blocks = 16,
97 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
98 }, {
99 .num_blocks = 8,
100 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
101 }, {
102 .num_blocks = 1,
103 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
104 } }
105};
106
107static const struct common_glue_ctx serpent_dec_cbc = {
108 .num_funcs = 3,
109 .fpu_blocks_limit = 8,
110
111 .funcs = { {
112 .num_blocks = 16,
113 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
114 }, {
115 .num_blocks = 8,
116 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
117 }, {
118 .num_blocks = 1,
119 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
120 } }
121};
122
123static const struct common_glue_ctx serpent_dec_xts = {
124 .num_funcs = 3,
125 .fpu_blocks_limit = 8,
126
127 .funcs = { {
128 .num_blocks = 16,
129 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
130 }, {
131 .num_blocks = 8,
132 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
133 }, {
134 .num_blocks = 1,
135 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
136 } }
137};
138
139static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
140 struct scatterlist *src, unsigned int nbytes)
141{
142 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
143}
144
145static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
146 struct scatterlist *src, unsigned int nbytes)
147{
148 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
149}
150
151static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
152 struct scatterlist *src, unsigned int nbytes)
153{
154 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
155 dst, src, nbytes);
156}
157
158static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
159 struct scatterlist *src, unsigned int nbytes)
160{
161 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
162 nbytes);
163}
164
165static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
166 struct scatterlist *src, unsigned int nbytes)
167{
168 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
169}
170
171static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
172{
173 /* since reusing AVX functions, starts using FPU at 8 parallel blocks */
174 return glue_fpu_begin(SERPENT_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes);
175}
176
177static inline void serpent_fpu_end(bool fpu_enabled)
178{
179 glue_fpu_end(fpu_enabled);
180}
181
182struct crypt_priv {
183 struct serpent_ctx *ctx;
184 bool fpu_enabled;
185};
186
187static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
188{
189 const unsigned int bsize = SERPENT_BLOCK_SIZE;
190 struct crypt_priv *ctx = priv;
191 int i;
192
193 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
194
195 if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
196 serpent_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
197 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
198 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
199 }
200
201 while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
202 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
203 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
204 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
205 }
206
207 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
208 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
209}
210
211static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
212{
213 const unsigned int bsize = SERPENT_BLOCK_SIZE;
214 struct crypt_priv *ctx = priv;
215 int i;
216
217 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
218
219 if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
220 serpent_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
221 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
222 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
223 }
224
225 while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
226 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
227 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
228 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
229 }
230
231 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
232 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
233}
234
235static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
236 struct scatterlist *src, unsigned int nbytes)
237{
238 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
239 be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
240 struct crypt_priv crypt_ctx = {
241 .ctx = &ctx->serpent_ctx,
242 .fpu_enabled = false,
243 };
244 struct lrw_crypt_req req = {
245 .tbuf = buf,
246 .tbuflen = sizeof(buf),
247
248 .table_ctx = &ctx->lrw_table,
249 .crypt_ctx = &crypt_ctx,
250 .crypt_fn = encrypt_callback,
251 };
252 int ret;
253
254 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
255 ret = lrw_crypt(desc, dst, src, nbytes, &req);
256 serpent_fpu_end(crypt_ctx.fpu_enabled);
257
258 return ret;
259}
260
261static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
262 struct scatterlist *src, unsigned int nbytes)
263{
264 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
265 be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
266 struct crypt_priv crypt_ctx = {
267 .ctx = &ctx->serpent_ctx,
268 .fpu_enabled = false,
269 };
270 struct lrw_crypt_req req = {
271 .tbuf = buf,
272 .tbuflen = sizeof(buf),
273
274 .table_ctx = &ctx->lrw_table,
275 .crypt_ctx = &crypt_ctx,
276 .crypt_fn = decrypt_callback,
277 };
278 int ret;
279
280 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
281 ret = lrw_crypt(desc, dst, src, nbytes, &req);
282 serpent_fpu_end(crypt_ctx.fpu_enabled);
283
284 return ret;
285}
286
287static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
288 struct scatterlist *src, unsigned int nbytes)
289{
290 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
291
292 return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes,
293 XTS_TWEAK_CAST(__serpent_encrypt),
294 &ctx->tweak_ctx, &ctx->crypt_ctx);
295}
296
297static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
298 struct scatterlist *src, unsigned int nbytes)
299{
300 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
301
302 return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes,
303 XTS_TWEAK_CAST(__serpent_encrypt),
304 &ctx->tweak_ctx, &ctx->crypt_ctx);
305}
306
307static struct crypto_alg srp_algs[10] = { {
308 .cra_name = "__ecb-serpent-avx2",
309 .cra_driver_name = "__driver-ecb-serpent-avx2",
310 .cra_priority = 0,
311 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
312 CRYPTO_ALG_INTERNAL,
313 .cra_blocksize = SERPENT_BLOCK_SIZE,
314 .cra_ctxsize = sizeof(struct serpent_ctx),
315 .cra_alignmask = 0,
316 .cra_type = &crypto_blkcipher_type,
317 .cra_module = THIS_MODULE,
318 .cra_list = LIST_HEAD_INIT(srp_algs[0].cra_list),
319 .cra_u = {
320 .blkcipher = {
321 .min_keysize = SERPENT_MIN_KEY_SIZE,
322 .max_keysize = SERPENT_MAX_KEY_SIZE,
323 .setkey = serpent_setkey,
324 .encrypt = ecb_encrypt,
325 .decrypt = ecb_decrypt,
326 },
327 },
328}, {
329 .cra_name = "__cbc-serpent-avx2",
330 .cra_driver_name = "__driver-cbc-serpent-avx2",
331 .cra_priority = 0,
332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
333 CRYPTO_ALG_INTERNAL,
334 .cra_blocksize = SERPENT_BLOCK_SIZE,
335 .cra_ctxsize = sizeof(struct serpent_ctx),
336 .cra_alignmask = 0,
337 .cra_type = &crypto_blkcipher_type,
338 .cra_module = THIS_MODULE,
339 .cra_list = LIST_HEAD_INIT(srp_algs[1].cra_list),
340 .cra_u = {
341 .blkcipher = {
342 .min_keysize = SERPENT_MIN_KEY_SIZE,
343 .max_keysize = SERPENT_MAX_KEY_SIZE,
344 .setkey = serpent_setkey,
345 .encrypt = cbc_encrypt,
346 .decrypt = cbc_decrypt,
347 },
348 },
349}, {
350 .cra_name = "__ctr-serpent-avx2",
351 .cra_driver_name = "__driver-ctr-serpent-avx2",
352 .cra_priority = 0,
353 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
354 CRYPTO_ALG_INTERNAL,
355 .cra_blocksize = 1,
356 .cra_ctxsize = sizeof(struct serpent_ctx),
357 .cra_alignmask = 0,
358 .cra_type = &crypto_blkcipher_type,
359 .cra_module = THIS_MODULE,
360 .cra_list = LIST_HEAD_INIT(srp_algs[2].cra_list),
361 .cra_u = {
362 .blkcipher = {
363 .min_keysize = SERPENT_MIN_KEY_SIZE,
364 .max_keysize = SERPENT_MAX_KEY_SIZE,
365 .ivsize = SERPENT_BLOCK_SIZE,
366 .setkey = serpent_setkey,
367 .encrypt = ctr_crypt,
368 .decrypt = ctr_crypt,
369 },
370 },
371}, {
372 .cra_name = "__lrw-serpent-avx2",
373 .cra_driver_name = "__driver-lrw-serpent-avx2",
374 .cra_priority = 0,
375 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
376 CRYPTO_ALG_INTERNAL,
377 .cra_blocksize = SERPENT_BLOCK_SIZE,
378 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
379 .cra_alignmask = 0,
380 .cra_type = &crypto_blkcipher_type,
381 .cra_module = THIS_MODULE,
382 .cra_list = LIST_HEAD_INIT(srp_algs[3].cra_list),
383 .cra_exit = lrw_serpent_exit_tfm,
384 .cra_u = {
385 .blkcipher = {
386 .min_keysize = SERPENT_MIN_KEY_SIZE +
387 SERPENT_BLOCK_SIZE,
388 .max_keysize = SERPENT_MAX_KEY_SIZE +
389 SERPENT_BLOCK_SIZE,
390 .ivsize = SERPENT_BLOCK_SIZE,
391 .setkey = lrw_serpent_setkey,
392 .encrypt = lrw_encrypt,
393 .decrypt = lrw_decrypt,
394 },
395 },
396}, {
397 .cra_name = "__xts-serpent-avx2",
398 .cra_driver_name = "__driver-xts-serpent-avx2",
399 .cra_priority = 0,
400 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
401 CRYPTO_ALG_INTERNAL,
402 .cra_blocksize = SERPENT_BLOCK_SIZE,
403 .cra_ctxsize = sizeof(struct serpent_xts_ctx),
404 .cra_alignmask = 0,
405 .cra_type = &crypto_blkcipher_type,
406 .cra_module = THIS_MODULE,
407 .cra_list = LIST_HEAD_INIT(srp_algs[4].cra_list),
408 .cra_u = {
409 .blkcipher = {
410 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
411 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
412 .ivsize = SERPENT_BLOCK_SIZE,
413 .setkey = xts_serpent_setkey,
414 .encrypt = xts_encrypt,
415 .decrypt = xts_decrypt,
416 },
417 },
418}, {
419 .cra_name = "ecb(serpent)",
420 .cra_driver_name = "ecb-serpent-avx2",
421 .cra_priority = 600,
422 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
423 .cra_blocksize = SERPENT_BLOCK_SIZE,
424 .cra_ctxsize = sizeof(struct async_helper_ctx),
425 .cra_alignmask = 0,
426 .cra_type = &crypto_ablkcipher_type,
427 .cra_module = THIS_MODULE,
428 .cra_list = LIST_HEAD_INIT(srp_algs[5].cra_list),
429 .cra_init = ablk_init,
430 .cra_exit = ablk_exit,
431 .cra_u = {
432 .ablkcipher = {
433 .min_keysize = SERPENT_MIN_KEY_SIZE,
434 .max_keysize = SERPENT_MAX_KEY_SIZE,
435 .setkey = ablk_set_key,
436 .encrypt = ablk_encrypt,
437 .decrypt = ablk_decrypt,
438 },
439 },
440}, {
441 .cra_name = "cbc(serpent)",
442 .cra_driver_name = "cbc-serpent-avx2",
443 .cra_priority = 600,
444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
445 .cra_blocksize = SERPENT_BLOCK_SIZE,
446 .cra_ctxsize = sizeof(struct async_helper_ctx),
447 .cra_alignmask = 0,
448 .cra_type = &crypto_ablkcipher_type,
449 .cra_module = THIS_MODULE,
450 .cra_list = LIST_HEAD_INIT(srp_algs[6].cra_list),
451 .cra_init = ablk_init,
452 .cra_exit = ablk_exit,
453 .cra_u = {
454 .ablkcipher = {
455 .min_keysize = SERPENT_MIN_KEY_SIZE,
456 .max_keysize = SERPENT_MAX_KEY_SIZE,
457 .ivsize = SERPENT_BLOCK_SIZE,
458 .setkey = ablk_set_key,
459 .encrypt = __ablk_encrypt,
460 .decrypt = ablk_decrypt,
461 },
462 },
463}, {
464 .cra_name = "ctr(serpent)",
465 .cra_driver_name = "ctr-serpent-avx2",
466 .cra_priority = 600,
467 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
468 .cra_blocksize = 1,
469 .cra_ctxsize = sizeof(struct async_helper_ctx),
470 .cra_alignmask = 0,
471 .cra_type = &crypto_ablkcipher_type,
472 .cra_module = THIS_MODULE,
473 .cra_list = LIST_HEAD_INIT(srp_algs[7].cra_list),
474 .cra_init = ablk_init,
475 .cra_exit = ablk_exit,
476 .cra_u = {
477 .ablkcipher = {
478 .min_keysize = SERPENT_MIN_KEY_SIZE,
479 .max_keysize = SERPENT_MAX_KEY_SIZE,
480 .ivsize = SERPENT_BLOCK_SIZE,
481 .setkey = ablk_set_key,
482 .encrypt = ablk_encrypt,
483 .decrypt = ablk_encrypt,
484 .geniv = "chainiv",
485 },
486 },
487}, {
488 .cra_name = "lrw(serpent)",
489 .cra_driver_name = "lrw-serpent-avx2",
490 .cra_priority = 600,
491 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
492 .cra_blocksize = SERPENT_BLOCK_SIZE,
493 .cra_ctxsize = sizeof(struct async_helper_ctx),
494 .cra_alignmask = 0,
495 .cra_type = &crypto_ablkcipher_type,
496 .cra_module = THIS_MODULE,
497 .cra_list = LIST_HEAD_INIT(srp_algs[8].cra_list),
498 .cra_init = ablk_init,
499 .cra_exit = ablk_exit,
500 .cra_u = {
501 .ablkcipher = {
502 .min_keysize = SERPENT_MIN_KEY_SIZE +
503 SERPENT_BLOCK_SIZE,
504 .max_keysize = SERPENT_MAX_KEY_SIZE +
505 SERPENT_BLOCK_SIZE,
506 .ivsize = SERPENT_BLOCK_SIZE,
507 .setkey = ablk_set_key,
508 .encrypt = ablk_encrypt,
509 .decrypt = ablk_decrypt,
510 },
511 },
512}, {
513 .cra_name = "xts(serpent)",
514 .cra_driver_name = "xts-serpent-avx2",
515 .cra_priority = 600,
516 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
517 .cra_blocksize = SERPENT_BLOCK_SIZE,
518 .cra_ctxsize = sizeof(struct async_helper_ctx),
519 .cra_alignmask = 0,
520 .cra_type = &crypto_ablkcipher_type,
521 .cra_module = THIS_MODULE,
522 .cra_list = LIST_HEAD_INIT(srp_algs[9].cra_list),
523 .cra_init = ablk_init,
524 .cra_exit = ablk_exit,
525 .cra_u = {
526 .ablkcipher = {
527 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
528 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
529 .ivsize = SERPENT_BLOCK_SIZE,
530 .setkey = ablk_set_key,
531 .encrypt = ablk_encrypt,
532 .decrypt = ablk_decrypt,
533 },
534 },
535} };
536
537static int __init init(void)
538{
539 const char *feature_name;
540
541 if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
542 pr_info("AVX2 instructions are not detected.\n");
543 return -ENODEV;
544 }
545 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
546 &feature_name)) {
547 pr_info("CPU feature '%s' is not supported.\n", feature_name);
548 return -ENODEV;
549 }
550
551 return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs));
552}
553
554static void __exit fini(void)
555{
556 crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs));
557}
558
559module_init(init);
560module_exit(fini);
561
562MODULE_LICENSE("GPL");
563MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
564MODULE_ALIAS_CRYPTO("serpent");
565MODULE_ALIAS_CRYPTO("serpent-asm");