Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26
27static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
28 unsigned int key_len)
29{
30 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
31
32 return sm4_expandkey(ctx, key, key_len);
33}
34
35static int cbc_decrypt(struct skcipher_request *req)
36{
37 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
38 sm4_aesni_avx2_cbc_dec_blk16);
39}
40
41
42static int ctr_crypt(struct skcipher_request *req)
43{
44 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
45 sm4_aesni_avx2_ctr_enc_blk16);
46}
47
48static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
49 {
50 .base = {
51 .cra_name = "__ecb(sm4)",
52 .cra_driver_name = "__ecb-sm4-aesni-avx2",
53 .cra_priority = 500,
54 .cra_flags = CRYPTO_ALG_INTERNAL,
55 .cra_blocksize = SM4_BLOCK_SIZE,
56 .cra_ctxsize = sizeof(struct sm4_ctx),
57 .cra_module = THIS_MODULE,
58 },
59 .min_keysize = SM4_KEY_SIZE,
60 .max_keysize = SM4_KEY_SIZE,
61 .walksize = 16 * SM4_BLOCK_SIZE,
62 .setkey = sm4_skcipher_setkey,
63 .encrypt = sm4_avx_ecb_encrypt,
64 .decrypt = sm4_avx_ecb_decrypt,
65 }, {
66 .base = {
67 .cra_name = "__cbc(sm4)",
68 .cra_driver_name = "__cbc-sm4-aesni-avx2",
69 .cra_priority = 500,
70 .cra_flags = CRYPTO_ALG_INTERNAL,
71 .cra_blocksize = SM4_BLOCK_SIZE,
72 .cra_ctxsize = sizeof(struct sm4_ctx),
73 .cra_module = THIS_MODULE,
74 },
75 .min_keysize = SM4_KEY_SIZE,
76 .max_keysize = SM4_KEY_SIZE,
77 .ivsize = SM4_BLOCK_SIZE,
78 .walksize = 16 * SM4_BLOCK_SIZE,
79 .setkey = sm4_skcipher_setkey,
80 .encrypt = sm4_cbc_encrypt,
81 .decrypt = cbc_decrypt,
82 }, {
83 .base = {
84 .cra_name = "__ctr(sm4)",
85 .cra_driver_name = "__ctr-sm4-aesni-avx2",
86 .cra_priority = 500,
87 .cra_flags = CRYPTO_ALG_INTERNAL,
88 .cra_blocksize = 1,
89 .cra_ctxsize = sizeof(struct sm4_ctx),
90 .cra_module = THIS_MODULE,
91 },
92 .min_keysize = SM4_KEY_SIZE,
93 .max_keysize = SM4_KEY_SIZE,
94 .ivsize = SM4_BLOCK_SIZE,
95 .chunksize = SM4_BLOCK_SIZE,
96 .walksize = 16 * SM4_BLOCK_SIZE,
97 .setkey = sm4_skcipher_setkey,
98 .encrypt = ctr_crypt,
99 .decrypt = ctr_crypt,
100 }
101};
102
103static struct simd_skcipher_alg *
104simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
105
106static int __init sm4_init(void)
107{
108 const char *feature_name;
109
110 if (!boot_cpu_has(X86_FEATURE_AVX) ||
111 !boot_cpu_has(X86_FEATURE_AVX2) ||
112 !boot_cpu_has(X86_FEATURE_AES) ||
113 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
114 pr_info("AVX2 or AES-NI instructions are not detected.\n");
115 return -ENODEV;
116 }
117
118 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
119 &feature_name)) {
120 pr_info("CPU feature '%s' is not supported.\n", feature_name);
121 return -ENODEV;
122 }
123
124 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
125 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
126 simd_sm4_aesni_avx2_skciphers);
127}
128
129static void __exit sm4_exit(void)
130{
131 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
132 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
133 simd_sm4_aesni_avx2_skciphers);
134}
135
136module_init(sm4_init);
137module_exit(sm4_exit);
138
139MODULE_LICENSE("GPL v2");
140MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
141MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
142MODULE_ALIAS_CRYPTO("sm4");
143MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26
27static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
28 unsigned int key_len)
29{
30 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
31
32 return sm4_expandkey(ctx, key, key_len);
33}
34
35static int cbc_decrypt(struct skcipher_request *req)
36{
37 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
38 sm4_aesni_avx2_cbc_dec_blk16);
39}
40
41
42static int ctr_crypt(struct skcipher_request *req)
43{
44 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
45 sm4_aesni_avx2_ctr_enc_blk16);
46}
47
48static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
49 {
50 .base = {
51 .cra_name = "__ecb(sm4)",
52 .cra_driver_name = "__ecb-sm4-aesni-avx2",
53 .cra_priority = 500,
54 .cra_flags = CRYPTO_ALG_INTERNAL,
55 .cra_blocksize = SM4_BLOCK_SIZE,
56 .cra_ctxsize = sizeof(struct sm4_ctx),
57 .cra_module = THIS_MODULE,
58 },
59 .min_keysize = SM4_KEY_SIZE,
60 .max_keysize = SM4_KEY_SIZE,
61 .walksize = 16 * SM4_BLOCK_SIZE,
62 .setkey = sm4_skcipher_setkey,
63 .encrypt = sm4_avx_ecb_encrypt,
64 .decrypt = sm4_avx_ecb_decrypt,
65 }, {
66 .base = {
67 .cra_name = "__cbc(sm4)",
68 .cra_driver_name = "__cbc-sm4-aesni-avx2",
69 .cra_priority = 500,
70 .cra_flags = CRYPTO_ALG_INTERNAL,
71 .cra_blocksize = SM4_BLOCK_SIZE,
72 .cra_ctxsize = sizeof(struct sm4_ctx),
73 .cra_module = THIS_MODULE,
74 },
75 .min_keysize = SM4_KEY_SIZE,
76 .max_keysize = SM4_KEY_SIZE,
77 .ivsize = SM4_BLOCK_SIZE,
78 .walksize = 16 * SM4_BLOCK_SIZE,
79 .setkey = sm4_skcipher_setkey,
80 .encrypt = sm4_cbc_encrypt,
81 .decrypt = cbc_decrypt,
82 }, {
83 .base = {
84 .cra_name = "__ctr(sm4)",
85 .cra_driver_name = "__ctr-sm4-aesni-avx2",
86 .cra_priority = 500,
87 .cra_flags = CRYPTO_ALG_INTERNAL,
88 .cra_blocksize = 1,
89 .cra_ctxsize = sizeof(struct sm4_ctx),
90 .cra_module = THIS_MODULE,
91 },
92 .min_keysize = SM4_KEY_SIZE,
93 .max_keysize = SM4_KEY_SIZE,
94 .ivsize = SM4_BLOCK_SIZE,
95 .chunksize = SM4_BLOCK_SIZE,
96 .walksize = 16 * SM4_BLOCK_SIZE,
97 .setkey = sm4_skcipher_setkey,
98 .encrypt = ctr_crypt,
99 .decrypt = ctr_crypt,
100 }
101};
102
103static struct simd_skcipher_alg *
104simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
105
106static int __init sm4_init(void)
107{
108 const char *feature_name;
109
110 if (!boot_cpu_has(X86_FEATURE_AVX) ||
111 !boot_cpu_has(X86_FEATURE_AVX2) ||
112 !boot_cpu_has(X86_FEATURE_AES) ||
113 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
114 pr_info("AVX2 or AES-NI instructions are not detected.\n");
115 return -ENODEV;
116 }
117
118 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
119 &feature_name)) {
120 pr_info("CPU feature '%s' is not supported.\n", feature_name);
121 return -ENODEV;
122 }
123
124 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
125 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
126 simd_sm4_aesni_avx2_skciphers);
127}
128
129static void __exit sm4_exit(void)
130{
131 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
132 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
133 simd_sm4_aesni_avx2_skciphers);
134}
135
136module_init(sm4_init);
137module_exit(sm4_exit);
138
139MODULE_LICENSE("GPL v2");
140MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
141MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
142MODULE_ALIAS_CRYPTO("sm4");
143MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");