Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
3 *
4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
5 */
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/crypto.h>
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/mm.h>
13#include <linux/types.h>
14#include <crypto/algapi.h>
15#include <crypto/internal/des.h>
16#include <crypto/internal/skcipher.h>
17
18#include <asm/fpumacro.h>
19#include <asm/pstate.h>
20#include <asm/elf.h>
21
22#include "opcodes.h"
23
24struct des_sparc64_ctx {
25 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
26 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
27};
28
29struct des3_ede_sparc64_ctx {
30 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
31 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
32};
33
34static void encrypt_to_decrypt(u64 *d, const u64 *e)
35{
36 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
37 int i;
38
39 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
40 *d++ = *s--;
41}
42
43extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
44
45static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
46 unsigned int keylen)
47{
48 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
49 int err;
50
51 /* Even though we have special instructions for key expansion,
52 * we call des_verify_key() so that we don't have to write our own
53 * weak key detection code.
54 */
55 err = crypto_des_verify_key(tfm, key);
56 if (err)
57 return err;
58
59 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
60 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
61
62 return 0;
63}
64
65static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
66 unsigned int keylen)
67{
68 return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
69}
70
71extern void des_sparc64_crypt(const u64 *key, const u64 *input,
72 u64 *output);
73
74static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
75{
76 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
77 const u64 *K = ctx->encrypt_expkey;
78
79 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
80}
81
82static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
83{
84 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
85 const u64 *K = ctx->decrypt_expkey;
86
87 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
88}
89
90extern void des_sparc64_load_keys(const u64 *key);
91
92extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
93 unsigned int len);
94
95static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
96{
97 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
98 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
99 struct skcipher_walk walk;
100 unsigned int nbytes;
101 int err;
102
103 err = skcipher_walk_virt(&walk, req, true);
104 if (err)
105 return err;
106
107 if (encrypt)
108 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
109 else
110 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
111 while ((nbytes = walk.nbytes) != 0) {
112 des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
113 round_down(nbytes, DES_BLOCK_SIZE));
114 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
115 }
116 fprs_write(0);
117 return err;
118}
119
120static int ecb_encrypt(struct skcipher_request *req)
121{
122 return __ecb_crypt(req, true);
123}
124
125static int ecb_decrypt(struct skcipher_request *req)
126{
127 return __ecb_crypt(req, false);
128}
129
130extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
131 unsigned int len, u64 *iv);
132
133extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
134 unsigned int len, u64 *iv);
135
136static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
137{
138 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
139 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
140 struct skcipher_walk walk;
141 unsigned int nbytes;
142 int err;
143
144 err = skcipher_walk_virt(&walk, req, true);
145 if (err)
146 return err;
147
148 if (encrypt)
149 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
150 else
151 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
152 while ((nbytes = walk.nbytes) != 0) {
153 if (encrypt)
154 des_sparc64_cbc_encrypt(walk.src.virt.addr,
155 walk.dst.virt.addr,
156 round_down(nbytes,
157 DES_BLOCK_SIZE),
158 walk.iv);
159 else
160 des_sparc64_cbc_decrypt(walk.src.virt.addr,
161 walk.dst.virt.addr,
162 round_down(nbytes,
163 DES_BLOCK_SIZE),
164 walk.iv);
165 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
166 }
167 fprs_write(0);
168 return err;
169}
170
171static int cbc_encrypt(struct skcipher_request *req)
172{
173 return __cbc_crypt(req, true);
174}
175
176static int cbc_decrypt(struct skcipher_request *req)
177{
178 return __cbc_crypt(req, false);
179}
180
181static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
182 unsigned int keylen)
183{
184 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
185 u64 k1[DES_EXPKEY_WORDS / 2];
186 u64 k2[DES_EXPKEY_WORDS / 2];
187 u64 k3[DES_EXPKEY_WORDS / 2];
188 int err;
189
190 err = crypto_des3_ede_verify_key(tfm, key);
191 if (err)
192 return err;
193
194 des_sparc64_key_expand((const u32 *)key, k1);
195 key += DES_KEY_SIZE;
196 des_sparc64_key_expand((const u32 *)key, k2);
197 key += DES_KEY_SIZE;
198 des_sparc64_key_expand((const u32 *)key, k3);
199
200 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
201 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
202 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
203 &k3[0], sizeof(k3));
204
205 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
206 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
207 &k2[0], sizeof(k2));
208 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
209 &k1[0]);
210
211 return 0;
212}
213
214static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
215 unsigned int keylen)
216{
217 return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
218}
219
220extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
221 u64 *output);
222
223static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
224{
225 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
226 const u64 *K = ctx->encrypt_expkey;
227
228 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
229}
230
231static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
232{
233 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
234 const u64 *K = ctx->decrypt_expkey;
235
236 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
237}
238
239extern void des3_ede_sparc64_load_keys(const u64 *key);
240
241extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
242 u64 *output, unsigned int len);
243
244static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
245{
246 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
247 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
248 struct skcipher_walk walk;
249 const u64 *K;
250 unsigned int nbytes;
251 int err;
252
253 err = skcipher_walk_virt(&walk, req, true);
254 if (err)
255 return err;
256
257 if (encrypt)
258 K = &ctx->encrypt_expkey[0];
259 else
260 K = &ctx->decrypt_expkey[0];
261 des3_ede_sparc64_load_keys(K);
262 while ((nbytes = walk.nbytes) != 0) {
263 des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
264 walk.dst.virt.addr,
265 round_down(nbytes, DES_BLOCK_SIZE));
266 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
267 }
268 fprs_write(0);
269 return err;
270}
271
272static int ecb3_encrypt(struct skcipher_request *req)
273{
274 return __ecb3_crypt(req, true);
275}
276
277static int ecb3_decrypt(struct skcipher_request *req)
278{
279 return __ecb3_crypt(req, false);
280}
281
282extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
283 u64 *output, unsigned int len,
284 u64 *iv);
285
286extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
287 u64 *output, unsigned int len,
288 u64 *iv);
289
290static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
291{
292 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
293 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
294 struct skcipher_walk walk;
295 const u64 *K;
296 unsigned int nbytes;
297 int err;
298
299 err = skcipher_walk_virt(&walk, req, true);
300 if (err)
301 return err;
302
303 if (encrypt)
304 K = &ctx->encrypt_expkey[0];
305 else
306 K = &ctx->decrypt_expkey[0];
307 des3_ede_sparc64_load_keys(K);
308 while ((nbytes = walk.nbytes) != 0) {
309 if (encrypt)
310 des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
311 walk.dst.virt.addr,
312 round_down(nbytes,
313 DES_BLOCK_SIZE),
314 walk.iv);
315 else
316 des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
317 walk.dst.virt.addr,
318 round_down(nbytes,
319 DES_BLOCK_SIZE),
320 walk.iv);
321 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
322 }
323 fprs_write(0);
324 return err;
325}
326
327static int cbc3_encrypt(struct skcipher_request *req)
328{
329 return __cbc3_crypt(req, true);
330}
331
332static int cbc3_decrypt(struct skcipher_request *req)
333{
334 return __cbc3_crypt(req, false);
335}
336
337static struct crypto_alg cipher_algs[] = {
338 {
339 .cra_name = "des",
340 .cra_driver_name = "des-sparc64",
341 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
342 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
343 .cra_blocksize = DES_BLOCK_SIZE,
344 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
345 .cra_alignmask = 7,
346 .cra_module = THIS_MODULE,
347 .cra_u = {
348 .cipher = {
349 .cia_min_keysize = DES_KEY_SIZE,
350 .cia_max_keysize = DES_KEY_SIZE,
351 .cia_setkey = des_set_key,
352 .cia_encrypt = sparc_des_encrypt,
353 .cia_decrypt = sparc_des_decrypt
354 }
355 }
356 }, {
357 .cra_name = "des3_ede",
358 .cra_driver_name = "des3_ede-sparc64",
359 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
360 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
361 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
362 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
363 .cra_alignmask = 7,
364 .cra_module = THIS_MODULE,
365 .cra_u = {
366 .cipher = {
367 .cia_min_keysize = DES3_EDE_KEY_SIZE,
368 .cia_max_keysize = DES3_EDE_KEY_SIZE,
369 .cia_setkey = des3_ede_set_key,
370 .cia_encrypt = sparc_des3_ede_encrypt,
371 .cia_decrypt = sparc_des3_ede_decrypt
372 }
373 }
374 }
375};
376
377static struct skcipher_alg skcipher_algs[] = {
378 {
379 .base.cra_name = "ecb(des)",
380 .base.cra_driver_name = "ecb-des-sparc64",
381 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
382 .base.cra_blocksize = DES_BLOCK_SIZE,
383 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
384 .base.cra_alignmask = 7,
385 .base.cra_module = THIS_MODULE,
386 .min_keysize = DES_KEY_SIZE,
387 .max_keysize = DES_KEY_SIZE,
388 .setkey = des_set_key_skcipher,
389 .encrypt = ecb_encrypt,
390 .decrypt = ecb_decrypt,
391 }, {
392 .base.cra_name = "cbc(des)",
393 .base.cra_driver_name = "cbc-des-sparc64",
394 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
395 .base.cra_blocksize = DES_BLOCK_SIZE,
396 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
397 .base.cra_alignmask = 7,
398 .base.cra_module = THIS_MODULE,
399 .min_keysize = DES_KEY_SIZE,
400 .max_keysize = DES_KEY_SIZE,
401 .ivsize = DES_BLOCK_SIZE,
402 .setkey = des_set_key_skcipher,
403 .encrypt = cbc_encrypt,
404 .decrypt = cbc_decrypt,
405 }, {
406 .base.cra_name = "ecb(des3_ede)",
407 .base.cra_driver_name = "ecb-des3_ede-sparc64",
408 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
409 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
410 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
411 .base.cra_alignmask = 7,
412 .base.cra_module = THIS_MODULE,
413 .min_keysize = DES3_EDE_KEY_SIZE,
414 .max_keysize = DES3_EDE_KEY_SIZE,
415 .setkey = des3_ede_set_key_skcipher,
416 .encrypt = ecb3_encrypt,
417 .decrypt = ecb3_decrypt,
418 }, {
419 .base.cra_name = "cbc(des3_ede)",
420 .base.cra_driver_name = "cbc-des3_ede-sparc64",
421 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
422 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
423 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
424 .base.cra_alignmask = 7,
425 .base.cra_module = THIS_MODULE,
426 .min_keysize = DES3_EDE_KEY_SIZE,
427 .max_keysize = DES3_EDE_KEY_SIZE,
428 .ivsize = DES3_EDE_BLOCK_SIZE,
429 .setkey = des3_ede_set_key_skcipher,
430 .encrypt = cbc3_encrypt,
431 .decrypt = cbc3_decrypt,
432 }
433};
434
435static bool __init sparc64_has_des_opcode(void)
436{
437 unsigned long cfr;
438
439 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
440 return false;
441
442 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
443 if (!(cfr & CFR_DES))
444 return false;
445
446 return true;
447}
448
449static int __init des_sparc64_mod_init(void)
450{
451 int err;
452
453 if (!sparc64_has_des_opcode()) {
454 pr_info("sparc64 des opcodes not available.\n");
455 return -ENODEV;
456 }
457 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
458 err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
459 if (err)
460 return err;
461 err = crypto_register_skciphers(skcipher_algs,
462 ARRAY_SIZE(skcipher_algs));
463 if (err)
464 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
465 return err;
466}
467
468static void __exit des_sparc64_mod_fini(void)
469{
470 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
471 crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
472}
473
474module_init(des_sparc64_mod_init);
475module_exit(des_sparc64_mod_fini);
476
477MODULE_LICENSE("GPL");
478MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
479
480MODULE_ALIAS_CRYPTO("des");
481MODULE_ALIAS_CRYPTO("des3_ede");
482
483#include "crop_devid.c"
1/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
2 *
3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/crypto.h>
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/mm.h>
12#include <linux/types.h>
13#include <crypto/algapi.h>
14#include <crypto/des.h>
15
16#include <asm/fpumacro.h>
17#include <asm/pstate.h>
18#include <asm/elf.h>
19
20#include "opcodes.h"
21
22struct des_sparc64_ctx {
23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
25};
26
27struct des3_ede_sparc64_ctx {
28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
30};
31
32static void encrypt_to_decrypt(u64 *d, const u64 *e)
33{
34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
35 int i;
36
37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
38 *d++ = *s--;
39}
40
41extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
42
43static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
44 unsigned int keylen)
45{
46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
47 u32 *flags = &tfm->crt_flags;
48 u32 tmp[DES_EXPKEY_WORDS];
49 int ret;
50
51 /* Even though we have special instructions for key expansion,
52 * we call des_ekey() so that we don't have to write our own
53 * weak key detection code.
54 */
55 ret = des_ekey(tmp, key);
56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
57 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
58 return -EINVAL;
59 }
60
61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
63
64 return 0;
65}
66
67extern void des_sparc64_crypt(const u64 *key, const u64 *input,
68 u64 *output);
69
70static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
71{
72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
73 const u64 *K = ctx->encrypt_expkey;
74
75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
76}
77
78static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
79{
80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
81 const u64 *K = ctx->decrypt_expkey;
82
83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
84}
85
86extern void des_sparc64_load_keys(const u64 *key);
87
88extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
89 unsigned int len);
90
91#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
92
93static int __ecb_crypt(struct blkcipher_desc *desc,
94 struct scatterlist *dst, struct scatterlist *src,
95 unsigned int nbytes, bool encrypt)
96{
97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
98 struct blkcipher_walk walk;
99 int err;
100
101 blkcipher_walk_init(&walk, dst, src, nbytes);
102 err = blkcipher_walk_virt(desc, &walk);
103 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
104
105 if (encrypt)
106 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
107 else
108 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
109 while ((nbytes = walk.nbytes)) {
110 unsigned int block_len = nbytes & DES_BLOCK_MASK;
111
112 if (likely(block_len)) {
113 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
114 (u64 *) walk.dst.virt.addr,
115 block_len);
116 }
117 nbytes &= DES_BLOCK_SIZE - 1;
118 err = blkcipher_walk_done(desc, &walk, nbytes);
119 }
120 fprs_write(0);
121 return err;
122}
123
124static int ecb_encrypt(struct blkcipher_desc *desc,
125 struct scatterlist *dst, struct scatterlist *src,
126 unsigned int nbytes)
127{
128 return __ecb_crypt(desc, dst, src, nbytes, true);
129}
130
131static int ecb_decrypt(struct blkcipher_desc *desc,
132 struct scatterlist *dst, struct scatterlist *src,
133 unsigned int nbytes)
134{
135 return __ecb_crypt(desc, dst, src, nbytes, false);
136}
137
138extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
139 unsigned int len, u64 *iv);
140
141static int cbc_encrypt(struct blkcipher_desc *desc,
142 struct scatterlist *dst, struct scatterlist *src,
143 unsigned int nbytes)
144{
145 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
146 struct blkcipher_walk walk;
147 int err;
148
149 blkcipher_walk_init(&walk, dst, src, nbytes);
150 err = blkcipher_walk_virt(desc, &walk);
151 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
152
153 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
154 while ((nbytes = walk.nbytes)) {
155 unsigned int block_len = nbytes & DES_BLOCK_MASK;
156
157 if (likely(block_len)) {
158 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
159 (u64 *) walk.dst.virt.addr,
160 block_len, (u64 *) walk.iv);
161 }
162 nbytes &= DES_BLOCK_SIZE - 1;
163 err = blkcipher_walk_done(desc, &walk, nbytes);
164 }
165 fprs_write(0);
166 return err;
167}
168
169extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
170 unsigned int len, u64 *iv);
171
172static int cbc_decrypt(struct blkcipher_desc *desc,
173 struct scatterlist *dst, struct scatterlist *src,
174 unsigned int nbytes)
175{
176 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
177 struct blkcipher_walk walk;
178 int err;
179
180 blkcipher_walk_init(&walk, dst, src, nbytes);
181 err = blkcipher_walk_virt(desc, &walk);
182 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
183
184 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
185 while ((nbytes = walk.nbytes)) {
186 unsigned int block_len = nbytes & DES_BLOCK_MASK;
187
188 if (likely(block_len)) {
189 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
190 (u64 *) walk.dst.virt.addr,
191 block_len, (u64 *) walk.iv);
192 }
193 nbytes &= DES_BLOCK_SIZE - 1;
194 err = blkcipher_walk_done(desc, &walk, nbytes);
195 }
196 fprs_write(0);
197 return err;
198}
199
200static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
201 unsigned int keylen)
202{
203 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
204 const u32 *K = (const u32 *)key;
205 u32 *flags = &tfm->crt_flags;
206 u64 k1[DES_EXPKEY_WORDS / 2];
207 u64 k2[DES_EXPKEY_WORDS / 2];
208 u64 k3[DES_EXPKEY_WORDS / 2];
209
210 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
211 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
212 (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
213 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
214 return -EINVAL;
215 }
216
217 des_sparc64_key_expand((const u32 *)key, k1);
218 key += DES_KEY_SIZE;
219 des_sparc64_key_expand((const u32 *)key, k2);
220 key += DES_KEY_SIZE;
221 des_sparc64_key_expand((const u32 *)key, k3);
222
223 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
224 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
225 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
226 &k3[0], sizeof(k3));
227
228 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
229 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
230 &k2[0], sizeof(k2));
231 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
232 &k1[0]);
233
234 return 0;
235}
236
237extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
238 u64 *output);
239
240static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
241{
242 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
243 const u64 *K = ctx->encrypt_expkey;
244
245 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
246}
247
248static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
249{
250 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
251 const u64 *K = ctx->decrypt_expkey;
252
253 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
254}
255
256extern void des3_ede_sparc64_load_keys(const u64 *key);
257
258extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
259 u64 *output, unsigned int len);
260
261static int __ecb3_crypt(struct blkcipher_desc *desc,
262 struct scatterlist *dst, struct scatterlist *src,
263 unsigned int nbytes, bool encrypt)
264{
265 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266 struct blkcipher_walk walk;
267 const u64 *K;
268 int err;
269
270 blkcipher_walk_init(&walk, dst, src, nbytes);
271 err = blkcipher_walk_virt(desc, &walk);
272 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
273
274 if (encrypt)
275 K = &ctx->encrypt_expkey[0];
276 else
277 K = &ctx->decrypt_expkey[0];
278 des3_ede_sparc64_load_keys(K);
279 while ((nbytes = walk.nbytes)) {
280 unsigned int block_len = nbytes & DES_BLOCK_MASK;
281
282 if (likely(block_len)) {
283 const u64 *src64 = (const u64 *)walk.src.virt.addr;
284 des3_ede_sparc64_ecb_crypt(K, src64,
285 (u64 *) walk.dst.virt.addr,
286 block_len);
287 }
288 nbytes &= DES_BLOCK_SIZE - 1;
289 err = blkcipher_walk_done(desc, &walk, nbytes);
290 }
291 fprs_write(0);
292 return err;
293}
294
295static int ecb3_encrypt(struct blkcipher_desc *desc,
296 struct scatterlist *dst, struct scatterlist *src,
297 unsigned int nbytes)
298{
299 return __ecb3_crypt(desc, dst, src, nbytes, true);
300}
301
302static int ecb3_decrypt(struct blkcipher_desc *desc,
303 struct scatterlist *dst, struct scatterlist *src,
304 unsigned int nbytes)
305{
306 return __ecb3_crypt(desc, dst, src, nbytes, false);
307}
308
309extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
310 u64 *output, unsigned int len,
311 u64 *iv);
312
313static int cbc3_encrypt(struct blkcipher_desc *desc,
314 struct scatterlist *dst, struct scatterlist *src,
315 unsigned int nbytes)
316{
317 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
318 struct blkcipher_walk walk;
319 const u64 *K;
320 int err;
321
322 blkcipher_walk_init(&walk, dst, src, nbytes);
323 err = blkcipher_walk_virt(desc, &walk);
324 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
325
326 K = &ctx->encrypt_expkey[0];
327 des3_ede_sparc64_load_keys(K);
328 while ((nbytes = walk.nbytes)) {
329 unsigned int block_len = nbytes & DES_BLOCK_MASK;
330
331 if (likely(block_len)) {
332 const u64 *src64 = (const u64 *)walk.src.virt.addr;
333 des3_ede_sparc64_cbc_encrypt(K, src64,
334 (u64 *) walk.dst.virt.addr,
335 block_len,
336 (u64 *) walk.iv);
337 }
338 nbytes &= DES_BLOCK_SIZE - 1;
339 err = blkcipher_walk_done(desc, &walk, nbytes);
340 }
341 fprs_write(0);
342 return err;
343}
344
345extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
346 u64 *output, unsigned int len,
347 u64 *iv);
348
349static int cbc3_decrypt(struct blkcipher_desc *desc,
350 struct scatterlist *dst, struct scatterlist *src,
351 unsigned int nbytes)
352{
353 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
354 struct blkcipher_walk walk;
355 const u64 *K;
356 int err;
357
358 blkcipher_walk_init(&walk, dst, src, nbytes);
359 err = blkcipher_walk_virt(desc, &walk);
360 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
361
362 K = &ctx->decrypt_expkey[0];
363 des3_ede_sparc64_load_keys(K);
364 while ((nbytes = walk.nbytes)) {
365 unsigned int block_len = nbytes & DES_BLOCK_MASK;
366
367 if (likely(block_len)) {
368 const u64 *src64 = (const u64 *)walk.src.virt.addr;
369 des3_ede_sparc64_cbc_decrypt(K, src64,
370 (u64 *) walk.dst.virt.addr,
371 block_len,
372 (u64 *) walk.iv);
373 }
374 nbytes &= DES_BLOCK_SIZE - 1;
375 err = blkcipher_walk_done(desc, &walk, nbytes);
376 }
377 fprs_write(0);
378 return err;
379}
380
381static struct crypto_alg algs[] = { {
382 .cra_name = "des",
383 .cra_driver_name = "des-sparc64",
384 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
385 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
386 .cra_blocksize = DES_BLOCK_SIZE,
387 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
388 .cra_alignmask = 7,
389 .cra_module = THIS_MODULE,
390 .cra_u = {
391 .cipher = {
392 .cia_min_keysize = DES_KEY_SIZE,
393 .cia_max_keysize = DES_KEY_SIZE,
394 .cia_setkey = des_set_key,
395 .cia_encrypt = des_encrypt,
396 .cia_decrypt = des_decrypt
397 }
398 }
399}, {
400 .cra_name = "ecb(des)",
401 .cra_driver_name = "ecb-des-sparc64",
402 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
403 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
404 .cra_blocksize = DES_BLOCK_SIZE,
405 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
406 .cra_alignmask = 7,
407 .cra_type = &crypto_blkcipher_type,
408 .cra_module = THIS_MODULE,
409 .cra_u = {
410 .blkcipher = {
411 .min_keysize = DES_KEY_SIZE,
412 .max_keysize = DES_KEY_SIZE,
413 .setkey = des_set_key,
414 .encrypt = ecb_encrypt,
415 .decrypt = ecb_decrypt,
416 },
417 },
418}, {
419 .cra_name = "cbc(des)",
420 .cra_driver_name = "cbc-des-sparc64",
421 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
422 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
423 .cra_blocksize = DES_BLOCK_SIZE,
424 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
425 .cra_alignmask = 7,
426 .cra_type = &crypto_blkcipher_type,
427 .cra_module = THIS_MODULE,
428 .cra_u = {
429 .blkcipher = {
430 .min_keysize = DES_KEY_SIZE,
431 .max_keysize = DES_KEY_SIZE,
432 .ivsize = DES_BLOCK_SIZE,
433 .setkey = des_set_key,
434 .encrypt = cbc_encrypt,
435 .decrypt = cbc_decrypt,
436 },
437 },
438}, {
439 .cra_name = "des3_ede",
440 .cra_driver_name = "des3_ede-sparc64",
441 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
442 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
443 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
444 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
445 .cra_alignmask = 7,
446 .cra_module = THIS_MODULE,
447 .cra_u = {
448 .cipher = {
449 .cia_min_keysize = DES3_EDE_KEY_SIZE,
450 .cia_max_keysize = DES3_EDE_KEY_SIZE,
451 .cia_setkey = des3_ede_set_key,
452 .cia_encrypt = des3_ede_encrypt,
453 .cia_decrypt = des3_ede_decrypt
454 }
455 }
456}, {
457 .cra_name = "ecb(des3_ede)",
458 .cra_driver_name = "ecb-des3_ede-sparc64",
459 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
460 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
461 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
462 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
463 .cra_alignmask = 7,
464 .cra_type = &crypto_blkcipher_type,
465 .cra_module = THIS_MODULE,
466 .cra_u = {
467 .blkcipher = {
468 .min_keysize = DES3_EDE_KEY_SIZE,
469 .max_keysize = DES3_EDE_KEY_SIZE,
470 .setkey = des3_ede_set_key,
471 .encrypt = ecb3_encrypt,
472 .decrypt = ecb3_decrypt,
473 },
474 },
475}, {
476 .cra_name = "cbc(des3_ede)",
477 .cra_driver_name = "cbc-des3_ede-sparc64",
478 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
479 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
480 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
481 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
482 .cra_alignmask = 7,
483 .cra_type = &crypto_blkcipher_type,
484 .cra_module = THIS_MODULE,
485 .cra_u = {
486 .blkcipher = {
487 .min_keysize = DES3_EDE_KEY_SIZE,
488 .max_keysize = DES3_EDE_KEY_SIZE,
489 .ivsize = DES3_EDE_BLOCK_SIZE,
490 .setkey = des3_ede_set_key,
491 .encrypt = cbc3_encrypt,
492 .decrypt = cbc3_decrypt,
493 },
494 },
495} };
496
497static bool __init sparc64_has_des_opcode(void)
498{
499 unsigned long cfr;
500
501 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
502 return false;
503
504 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
505 if (!(cfr & CFR_DES))
506 return false;
507
508 return true;
509}
510
511static int __init des_sparc64_mod_init(void)
512{
513 int i;
514
515 for (i = 0; i < ARRAY_SIZE(algs); i++)
516 INIT_LIST_HEAD(&algs[i].cra_list);
517
518 if (sparc64_has_des_opcode()) {
519 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
520 return crypto_register_algs(algs, ARRAY_SIZE(algs));
521 }
522 pr_info("sparc64 des opcodes not available.\n");
523 return -ENODEV;
524}
525
526static void __exit des_sparc64_mod_fini(void)
527{
528 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
529}
530
531module_init(des_sparc64_mod_init);
532module_exit(des_sparc64_mod_fini);
533
534MODULE_LICENSE("GPL");
535MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
536
537MODULE_ALIAS_CRYPTO("des");
538MODULE_ALIAS_CRYPTO("des3_ede");
539
540#include "crop_devid.c"