Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Crypto-API module for CRC-32 algorithms implemented with the
4 * z/Architecture Vector Extension Facility.
5 *
6 * Copyright IBM Corp. 2015
7 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8 */
9#define KMSG_COMPONENT "crc32-vx"
10#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11
12#include <linux/module.h>
13#include <linux/cpufeature.h>
14#include <linux/crc32.h>
15#include <crypto/internal/hash.h>
16#include <asm/fpu/api.h>
17
18
19#define CRC32_BLOCK_SIZE 1
20#define CRC32_DIGEST_SIZE 4
21
22#define VX_MIN_LEN 64
23#define VX_ALIGNMENT 16L
24#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
25
26struct crc_ctx {
27 u32 key;
28};
29
30struct crc_desc_ctx {
31 u32 crc;
32};
33
34/* Prototypes for functions in assembly files */
35u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
36u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
37u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
38
39/*
40 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
41 *
42 * Creates a function to perform a particular CRC-32 computation. Depending
43 * on the message buffer, the hardware-accelerated or software implementation
44 * is used. Note that the message buffer is aligned to improve fetch
45 * operations of VECTOR LOAD MULTIPLE instructions.
46 *
47 */
48#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
49 static u32 __pure ___fname(u32 crc, \
50 unsigned char const *data, size_t datalen) \
51 { \
52 struct kernel_fpu vxstate; \
53 unsigned long prealign, aligned, remaining; \
54 \
55 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
56 return ___crc32_sw(crc, data, datalen); \
57 \
58 if ((unsigned long)data & VX_ALIGN_MASK) { \
59 prealign = VX_ALIGNMENT - \
60 ((unsigned long)data & VX_ALIGN_MASK); \
61 datalen -= prealign; \
62 crc = ___crc32_sw(crc, data, prealign); \
63 data = (void *)((unsigned long)data + prealign); \
64 } \
65 \
66 aligned = datalen & ~VX_ALIGN_MASK; \
67 remaining = datalen & VX_ALIGN_MASK; \
68 \
69 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
70 crc = ___crc32_vx(crc, data, aligned); \
71 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
72 \
73 if (remaining) \
74 crc = ___crc32_sw(crc, data + aligned, remaining); \
75 \
76 return crc; \
77 }
78
79DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
80DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
81DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
82
83
84static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
85{
86 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
87
88 mctx->key = 0;
89 return 0;
90}
91
92static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
93{
94 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
95
96 mctx->key = ~0;
97 return 0;
98}
99
100static int crc32_vx_init(struct shash_desc *desc)
101{
102 struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
103 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
104
105 ctx->crc = mctx->key;
106 return 0;
107}
108
109static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
110 unsigned int newkeylen)
111{
112 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
113
114 if (newkeylen != sizeof(mctx->key))
115 return -EINVAL;
116 mctx->key = le32_to_cpu(*(__le32 *)newkey);
117 return 0;
118}
119
120static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
121 unsigned int newkeylen)
122{
123 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
124
125 if (newkeylen != sizeof(mctx->key))
126 return -EINVAL;
127 mctx->key = be32_to_cpu(*(__be32 *)newkey);
128 return 0;
129}
130
131static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
132{
133 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
134
135 *(__le32 *)out = cpu_to_le32p(&ctx->crc);
136 return 0;
137}
138
139static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
140{
141 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
142
143 *(__be32 *)out = cpu_to_be32p(&ctx->crc);
144 return 0;
145}
146
147static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
148{
149 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
150
151 /*
152 * Perform a final XOR with 0xFFFFFFFF to be in sync
153 * with the generic crc32c shash implementation.
154 */
155 *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
156 return 0;
157}
158
159static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
160 u8 *out)
161{
162 *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
163 return 0;
164}
165
166static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
167 u8 *out)
168{
169 *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
170 return 0;
171}
172
173static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
174 u8 *out)
175{
176 /*
177 * Perform a final XOR with 0xFFFFFFFF to be in sync
178 * with the generic crc32c shash implementation.
179 */
180 *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
181 return 0;
182}
183
184
185#define CRC32_VX_FINUP(alg, func) \
186 static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
187 unsigned int datalen, u8 *out) \
188 { \
189 return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
190 data, datalen, out); \
191 }
192
193CRC32_VX_FINUP(crc32le, crc32_le_vx)
194CRC32_VX_FINUP(crc32be, crc32_be_vx)
195CRC32_VX_FINUP(crc32c, crc32c_le_vx)
196
197#define CRC32_VX_DIGEST(alg, func) \
198 static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
199 unsigned int len, u8 *out) \
200 { \
201 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
202 data, len, out); \
203 }
204
205CRC32_VX_DIGEST(crc32le, crc32_le_vx)
206CRC32_VX_DIGEST(crc32be, crc32_be_vx)
207CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
208
209#define CRC32_VX_UPDATE(alg, func) \
210 static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
211 unsigned int datalen) \
212 { \
213 struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
214 ctx->crc = func(ctx->crc, data, datalen); \
215 return 0; \
216 }
217
218CRC32_VX_UPDATE(crc32le, crc32_le_vx)
219CRC32_VX_UPDATE(crc32be, crc32_be_vx)
220CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
221
222
223static struct shash_alg crc32_vx_algs[] = {
224 /* CRC-32 LE */
225 {
226 .init = crc32_vx_init,
227 .setkey = crc32_vx_setkey,
228 .update = crc32le_vx_update,
229 .final = crc32le_vx_final,
230 .finup = crc32le_vx_finup,
231 .digest = crc32le_vx_digest,
232 .descsize = sizeof(struct crc_desc_ctx),
233 .digestsize = CRC32_DIGEST_SIZE,
234 .base = {
235 .cra_name = "crc32",
236 .cra_driver_name = "crc32-vx",
237 .cra_priority = 200,
238 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
239 .cra_blocksize = CRC32_BLOCK_SIZE,
240 .cra_ctxsize = sizeof(struct crc_ctx),
241 .cra_module = THIS_MODULE,
242 .cra_init = crc32_vx_cra_init_zero,
243 },
244 },
245 /* CRC-32 BE */
246 {
247 .init = crc32_vx_init,
248 .setkey = crc32be_vx_setkey,
249 .update = crc32be_vx_update,
250 .final = crc32be_vx_final,
251 .finup = crc32be_vx_finup,
252 .digest = crc32be_vx_digest,
253 .descsize = sizeof(struct crc_desc_ctx),
254 .digestsize = CRC32_DIGEST_SIZE,
255 .base = {
256 .cra_name = "crc32be",
257 .cra_driver_name = "crc32be-vx",
258 .cra_priority = 200,
259 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
260 .cra_blocksize = CRC32_BLOCK_SIZE,
261 .cra_ctxsize = sizeof(struct crc_ctx),
262 .cra_module = THIS_MODULE,
263 .cra_init = crc32_vx_cra_init_zero,
264 },
265 },
266 /* CRC-32C LE */
267 {
268 .init = crc32_vx_init,
269 .setkey = crc32_vx_setkey,
270 .update = crc32c_vx_update,
271 .final = crc32c_vx_final,
272 .finup = crc32c_vx_finup,
273 .digest = crc32c_vx_digest,
274 .descsize = sizeof(struct crc_desc_ctx),
275 .digestsize = CRC32_DIGEST_SIZE,
276 .base = {
277 .cra_name = "crc32c",
278 .cra_driver_name = "crc32c-vx",
279 .cra_priority = 200,
280 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
281 .cra_blocksize = CRC32_BLOCK_SIZE,
282 .cra_ctxsize = sizeof(struct crc_ctx),
283 .cra_module = THIS_MODULE,
284 .cra_init = crc32_vx_cra_init_invert,
285 },
286 },
287};
288
289
290static int __init crc_vx_mod_init(void)
291{
292 return crypto_register_shashes(crc32_vx_algs,
293 ARRAY_SIZE(crc32_vx_algs));
294}
295
296static void __exit crc_vx_mod_exit(void)
297{
298 crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
299}
300
301module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
302module_exit(crc_vx_mod_exit);
303
304MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
305MODULE_LICENSE("GPL");
306
307MODULE_ALIAS_CRYPTO("crc32");
308MODULE_ALIAS_CRYPTO("crc32-vx");
309MODULE_ALIAS_CRYPTO("crc32c");
310MODULE_ALIAS_CRYPTO("crc32c-vx");
1/*
2 * Crypto-API module for CRC-32 algorithms implemented with the
3 * z/Architecture Vector Extension Facility.
4 *
5 * Copyright IBM Corp. 2015
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7 */
8#define KMSG_COMPONENT "crc32-vx"
9#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11#include <linux/module.h>
12#include <linux/cpufeature.h>
13#include <linux/crc32.h>
14#include <crypto/internal/hash.h>
15#include <asm/fpu/api.h>
16
17
18#define CRC32_BLOCK_SIZE 1
19#define CRC32_DIGEST_SIZE 4
20
21#define VX_MIN_LEN 64
22#define VX_ALIGNMENT 16L
23#define VX_ALIGN_MASK (VX_ALIGNMENT - 1)
24
25struct crc_ctx {
26 u32 key;
27};
28
29struct crc_desc_ctx {
30 u32 crc;
31};
32
33/* Prototypes for functions in assembly files */
34u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
35u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
36u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
37
38/*
39 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
40 *
41 * Creates a function to perform a particular CRC-32 computation. Depending
42 * on the message buffer, the hardware-accelerated or software implementation
43 * is used. Note that the message buffer is aligned to improve fetch
44 * operations of VECTOR LOAD MULTIPLE instructions.
45 *
46 */
47#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
48 static u32 __pure ___fname(u32 crc, \
49 unsigned char const *data, size_t datalen) \
50 { \
51 struct kernel_fpu vxstate; \
52 unsigned long prealign, aligned, remaining; \
53 \
54 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \
55 return ___crc32_sw(crc, data, datalen); \
56 \
57 if ((unsigned long)data & VX_ALIGN_MASK) { \
58 prealign = VX_ALIGNMENT - \
59 ((unsigned long)data & VX_ALIGN_MASK); \
60 datalen -= prealign; \
61 crc = ___crc32_sw(crc, data, prealign); \
62 data = (void *)((unsigned long)data + prealign); \
63 } \
64 \
65 aligned = datalen & ~VX_ALIGN_MASK; \
66 remaining = datalen & VX_ALIGN_MASK; \
67 \
68 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \
69 crc = ___crc32_vx(crc, data, aligned); \
70 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \
71 \
72 if (remaining) \
73 crc = ___crc32_sw(crc, data + aligned, remaining); \
74 \
75 return crc; \
76 }
77
78DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
79DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
80DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
81
82
83static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
84{
85 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
86
87 mctx->key = 0;
88 return 0;
89}
90
91static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
92{
93 struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
94
95 mctx->key = ~0;
96 return 0;
97}
98
99static int crc32_vx_init(struct shash_desc *desc)
100{
101 struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
102 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
103
104 ctx->crc = mctx->key;
105 return 0;
106}
107
108static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
109 unsigned int newkeylen)
110{
111 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
112
113 if (newkeylen != sizeof(mctx->key)) {
114 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
115 return -EINVAL;
116 }
117 mctx->key = le32_to_cpu(*(__le32 *)newkey);
118 return 0;
119}
120
121static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
122 unsigned int newkeylen)
123{
124 struct crc_ctx *mctx = crypto_shash_ctx(tfm);
125
126 if (newkeylen != sizeof(mctx->key)) {
127 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
128 return -EINVAL;
129 }
130 mctx->key = be32_to_cpu(*(__be32 *)newkey);
131 return 0;
132}
133
134static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
135{
136 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
137
138 *(__le32 *)out = cpu_to_le32p(&ctx->crc);
139 return 0;
140}
141
142static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
143{
144 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
145
146 *(__be32 *)out = cpu_to_be32p(&ctx->crc);
147 return 0;
148}
149
150static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
151{
152 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
153
154 /*
155 * Perform a final XOR with 0xFFFFFFFF to be in sync
156 * with the generic crc32c shash implementation.
157 */
158 *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
159 return 0;
160}
161
162static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
163 u8 *out)
164{
165 *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
166 return 0;
167}
168
169static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
170 u8 *out)
171{
172 *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
173 return 0;
174}
175
176static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
177 u8 *out)
178{
179 /*
180 * Perform a final XOR with 0xFFFFFFFF to be in sync
181 * with the generic crc32c shash implementation.
182 */
183 *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
184 return 0;
185}
186
187
188#define CRC32_VX_FINUP(alg, func) \
189 static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data, \
190 unsigned int datalen, u8 *out) \
191 { \
192 return __ ## alg ## _vx_finup(shash_desc_ctx(desc), \
193 data, datalen, out); \
194 }
195
196CRC32_VX_FINUP(crc32le, crc32_le_vx)
197CRC32_VX_FINUP(crc32be, crc32_be_vx)
198CRC32_VX_FINUP(crc32c, crc32c_le_vx)
199
200#define CRC32_VX_DIGEST(alg, func) \
201 static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
202 unsigned int len, u8 *out) \
203 { \
204 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm), \
205 data, len, out); \
206 }
207
208CRC32_VX_DIGEST(crc32le, crc32_le_vx)
209CRC32_VX_DIGEST(crc32be, crc32_be_vx)
210CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
211
212#define CRC32_VX_UPDATE(alg, func) \
213 static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
214 unsigned int datalen) \
215 { \
216 struct crc_desc_ctx *ctx = shash_desc_ctx(desc); \
217 ctx->crc = func(ctx->crc, data, datalen); \
218 return 0; \
219 }
220
221CRC32_VX_UPDATE(crc32le, crc32_le_vx)
222CRC32_VX_UPDATE(crc32be, crc32_be_vx)
223CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
224
225
226static struct shash_alg crc32_vx_algs[] = {
227 /* CRC-32 LE */
228 {
229 .init = crc32_vx_init,
230 .setkey = crc32_vx_setkey,
231 .update = crc32le_vx_update,
232 .final = crc32le_vx_final,
233 .finup = crc32le_vx_finup,
234 .digest = crc32le_vx_digest,
235 .descsize = sizeof(struct crc_desc_ctx),
236 .digestsize = CRC32_DIGEST_SIZE,
237 .base = {
238 .cra_name = "crc32",
239 .cra_driver_name = "crc32-vx",
240 .cra_priority = 200,
241 .cra_blocksize = CRC32_BLOCK_SIZE,
242 .cra_ctxsize = sizeof(struct crc_ctx),
243 .cra_module = THIS_MODULE,
244 .cra_init = crc32_vx_cra_init_zero,
245 },
246 },
247 /* CRC-32 BE */
248 {
249 .init = crc32_vx_init,
250 .setkey = crc32be_vx_setkey,
251 .update = crc32be_vx_update,
252 .final = crc32be_vx_final,
253 .finup = crc32be_vx_finup,
254 .digest = crc32be_vx_digest,
255 .descsize = sizeof(struct crc_desc_ctx),
256 .digestsize = CRC32_DIGEST_SIZE,
257 .base = {
258 .cra_name = "crc32be",
259 .cra_driver_name = "crc32be-vx",
260 .cra_priority = 200,
261 .cra_blocksize = CRC32_BLOCK_SIZE,
262 .cra_ctxsize = sizeof(struct crc_ctx),
263 .cra_module = THIS_MODULE,
264 .cra_init = crc32_vx_cra_init_zero,
265 },
266 },
267 /* CRC-32C LE */
268 {
269 .init = crc32_vx_init,
270 .setkey = crc32_vx_setkey,
271 .update = crc32c_vx_update,
272 .final = crc32c_vx_final,
273 .finup = crc32c_vx_finup,
274 .digest = crc32c_vx_digest,
275 .descsize = sizeof(struct crc_desc_ctx),
276 .digestsize = CRC32_DIGEST_SIZE,
277 .base = {
278 .cra_name = "crc32c",
279 .cra_driver_name = "crc32c-vx",
280 .cra_priority = 200,
281 .cra_blocksize = CRC32_BLOCK_SIZE,
282 .cra_ctxsize = sizeof(struct crc_ctx),
283 .cra_module = THIS_MODULE,
284 .cra_init = crc32_vx_cra_init_invert,
285 },
286 },
287};
288
289
290static int __init crc_vx_mod_init(void)
291{
292 return crypto_register_shashes(crc32_vx_algs,
293 ARRAY_SIZE(crc32_vx_algs));
294}
295
296static void __exit crc_vx_mod_exit(void)
297{
298 crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
299}
300
301module_cpu_feature_match(VXRS, crc_vx_mod_init);
302module_exit(crc_vx_mod_exit);
303
304MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
305MODULE_LICENSE("GPL");
306
307MODULE_ALIAS_CRYPTO("crc32");
308MODULE_ALIAS_CRYPTO("crc32-vx");
309MODULE_ALIAS_CRYPTO("crc32c");
310MODULE_ALIAS_CRYPTO("crc32c-vx");