Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26
27static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
28 unsigned int key_len)
29{
30 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
31
32 return sm4_expandkey(ctx, key, key_len);
33}
34
35static int cbc_decrypt(struct skcipher_request *req)
36{
37 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
38 sm4_aesni_avx2_cbc_dec_blk16);
39}
40
41
42static int ctr_crypt(struct skcipher_request *req)
43{
44 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
45 sm4_aesni_avx2_ctr_enc_blk16);
46}
47
48static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
49 {
50 .base = {
51 .cra_name = "__ecb(sm4)",
52 .cra_driver_name = "__ecb-sm4-aesni-avx2",
53 .cra_priority = 500,
54 .cra_flags = CRYPTO_ALG_INTERNAL,
55 .cra_blocksize = SM4_BLOCK_SIZE,
56 .cra_ctxsize = sizeof(struct sm4_ctx),
57 .cra_module = THIS_MODULE,
58 },
59 .min_keysize = SM4_KEY_SIZE,
60 .max_keysize = SM4_KEY_SIZE,
61 .walksize = 16 * SM4_BLOCK_SIZE,
62 .setkey = sm4_skcipher_setkey,
63 .encrypt = sm4_avx_ecb_encrypt,
64 .decrypt = sm4_avx_ecb_decrypt,
65 }, {
66 .base = {
67 .cra_name = "__cbc(sm4)",
68 .cra_driver_name = "__cbc-sm4-aesni-avx2",
69 .cra_priority = 500,
70 .cra_flags = CRYPTO_ALG_INTERNAL,
71 .cra_blocksize = SM4_BLOCK_SIZE,
72 .cra_ctxsize = sizeof(struct sm4_ctx),
73 .cra_module = THIS_MODULE,
74 },
75 .min_keysize = SM4_KEY_SIZE,
76 .max_keysize = SM4_KEY_SIZE,
77 .ivsize = SM4_BLOCK_SIZE,
78 .walksize = 16 * SM4_BLOCK_SIZE,
79 .setkey = sm4_skcipher_setkey,
80 .encrypt = sm4_cbc_encrypt,
81 .decrypt = cbc_decrypt,
82 }, {
83 .base = {
84 .cra_name = "__ctr(sm4)",
85 .cra_driver_name = "__ctr-sm4-aesni-avx2",
86 .cra_priority = 500,
87 .cra_flags = CRYPTO_ALG_INTERNAL,
88 .cra_blocksize = 1,
89 .cra_ctxsize = sizeof(struct sm4_ctx),
90 .cra_module = THIS_MODULE,
91 },
92 .min_keysize = SM4_KEY_SIZE,
93 .max_keysize = SM4_KEY_SIZE,
94 .ivsize = SM4_BLOCK_SIZE,
95 .chunksize = SM4_BLOCK_SIZE,
96 .walksize = 16 * SM4_BLOCK_SIZE,
97 .setkey = sm4_skcipher_setkey,
98 .encrypt = ctr_crypt,
99 .decrypt = ctr_crypt,
100 }
101};
102
103static struct simd_skcipher_alg *
104simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
105
106static int __init sm4_init(void)
107{
108 const char *feature_name;
109
110 if (!boot_cpu_has(X86_FEATURE_AVX) ||
111 !boot_cpu_has(X86_FEATURE_AVX2) ||
112 !boot_cpu_has(X86_FEATURE_AES) ||
113 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
114 pr_info("AVX2 or AES-NI instructions are not detected.\n");
115 return -ENODEV;
116 }
117
118 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
119 &feature_name)) {
120 pr_info("CPU feature '%s' is not supported.\n", feature_name);
121 return -ENODEV;
122 }
123
124 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
125 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
126 simd_sm4_aesni_avx2_skciphers);
127}
128
129static void __exit sm4_exit(void)
130{
131 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
132 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
133 simd_sm4_aesni_avx2_skciphers);
134}
135
136module_init(sm4_init);
137module_exit(sm4_exit);
138
139MODULE_LICENSE("GPL v2");
140MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
141MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
142MODULE_ALIAS_CRYPTO("sm4");
143MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
27 const u8 *src, u8 *iv);
28
29static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
30 unsigned int key_len)
31{
32 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
33
34 return sm4_expandkey(ctx, key, key_len);
35}
36
37static int cbc_decrypt(struct skcipher_request *req)
38{
39 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
40 sm4_aesni_avx2_cbc_dec_blk16);
41}
42
43
44static int cfb_decrypt(struct skcipher_request *req)
45{
46 return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
47 sm4_aesni_avx2_cfb_dec_blk16);
48}
49
50static int ctr_crypt(struct skcipher_request *req)
51{
52 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
53 sm4_aesni_avx2_ctr_enc_blk16);
54}
55
56static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
57 {
58 .base = {
59 .cra_name = "__ecb(sm4)",
60 .cra_driver_name = "__ecb-sm4-aesni-avx2",
61 .cra_priority = 500,
62 .cra_flags = CRYPTO_ALG_INTERNAL,
63 .cra_blocksize = SM4_BLOCK_SIZE,
64 .cra_ctxsize = sizeof(struct sm4_ctx),
65 .cra_module = THIS_MODULE,
66 },
67 .min_keysize = SM4_KEY_SIZE,
68 .max_keysize = SM4_KEY_SIZE,
69 .walksize = 16 * SM4_BLOCK_SIZE,
70 .setkey = sm4_skcipher_setkey,
71 .encrypt = sm4_avx_ecb_encrypt,
72 .decrypt = sm4_avx_ecb_decrypt,
73 }, {
74 .base = {
75 .cra_name = "__cbc(sm4)",
76 .cra_driver_name = "__cbc-sm4-aesni-avx2",
77 .cra_priority = 500,
78 .cra_flags = CRYPTO_ALG_INTERNAL,
79 .cra_blocksize = SM4_BLOCK_SIZE,
80 .cra_ctxsize = sizeof(struct sm4_ctx),
81 .cra_module = THIS_MODULE,
82 },
83 .min_keysize = SM4_KEY_SIZE,
84 .max_keysize = SM4_KEY_SIZE,
85 .ivsize = SM4_BLOCK_SIZE,
86 .walksize = 16 * SM4_BLOCK_SIZE,
87 .setkey = sm4_skcipher_setkey,
88 .encrypt = sm4_cbc_encrypt,
89 .decrypt = cbc_decrypt,
90 }, {
91 .base = {
92 .cra_name = "__cfb(sm4)",
93 .cra_driver_name = "__cfb-sm4-aesni-avx2",
94 .cra_priority = 500,
95 .cra_flags = CRYPTO_ALG_INTERNAL,
96 .cra_blocksize = 1,
97 .cra_ctxsize = sizeof(struct sm4_ctx),
98 .cra_module = THIS_MODULE,
99 },
100 .min_keysize = SM4_KEY_SIZE,
101 .max_keysize = SM4_KEY_SIZE,
102 .ivsize = SM4_BLOCK_SIZE,
103 .chunksize = SM4_BLOCK_SIZE,
104 .walksize = 16 * SM4_BLOCK_SIZE,
105 .setkey = sm4_skcipher_setkey,
106 .encrypt = sm4_cfb_encrypt,
107 .decrypt = cfb_decrypt,
108 }, {
109 .base = {
110 .cra_name = "__ctr(sm4)",
111 .cra_driver_name = "__ctr-sm4-aesni-avx2",
112 .cra_priority = 500,
113 .cra_flags = CRYPTO_ALG_INTERNAL,
114 .cra_blocksize = 1,
115 .cra_ctxsize = sizeof(struct sm4_ctx),
116 .cra_module = THIS_MODULE,
117 },
118 .min_keysize = SM4_KEY_SIZE,
119 .max_keysize = SM4_KEY_SIZE,
120 .ivsize = SM4_BLOCK_SIZE,
121 .chunksize = SM4_BLOCK_SIZE,
122 .walksize = 16 * SM4_BLOCK_SIZE,
123 .setkey = sm4_skcipher_setkey,
124 .encrypt = ctr_crypt,
125 .decrypt = ctr_crypt,
126 }
127};
128
129static struct simd_skcipher_alg *
130simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
131
132static int __init sm4_init(void)
133{
134 const char *feature_name;
135
136 if (!boot_cpu_has(X86_FEATURE_AVX) ||
137 !boot_cpu_has(X86_FEATURE_AVX2) ||
138 !boot_cpu_has(X86_FEATURE_AES) ||
139 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
140 pr_info("AVX2 or AES-NI instructions are not detected.\n");
141 return -ENODEV;
142 }
143
144 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
145 &feature_name)) {
146 pr_info("CPU feature '%s' is not supported.\n", feature_name);
147 return -ENODEV;
148 }
149
150 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
151 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
152 simd_sm4_aesni_avx2_skciphers);
153}
154
155static void __exit sm4_exit(void)
156{
157 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
158 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
159 simd_sm4_aesni_avx2_skciphers);
160}
161
162module_init(sm4_init);
163module_exit(sm4_exit);
164
165MODULE_LICENSE("GPL v2");
166MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
167MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
168MODULE_ALIAS_CRYPTO("sm4");
169MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");