Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Cryptographic Coprocessor (CCP) driver
4 *
5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
9 */
10
11#include <linux/dma-mapping.h>
12#include <linux/module.h>
13#include <linux/kernel.h>
14#include <linux/interrupt.h>
15#include <crypto/scatterwalk.h>
16#include <crypto/des.h>
17#include <linux/ccp.h>
18
19#include "ccp-dev.h"
20
21/* SHA initial context values */
22static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
25 cpu_to_be32(SHA1_H4),
26};
27
28static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33};
34
35static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40};
41
42static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47};
48
49static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54};
55
56#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
57 ccp_gen_jobid(ccp) : 0)
58
59static u32 ccp_gen_jobid(struct ccp_device *ccp)
60{
61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
62}
63
64static void ccp_sg_free(struct ccp_sg_workarea *wa)
65{
66 if (wa->dma_count)
67 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
68
69 wa->dma_count = 0;
70}
71
72static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
73 struct scatterlist *sg, u64 len,
74 enum dma_data_direction dma_dir)
75{
76 memset(wa, 0, sizeof(*wa));
77
78 wa->sg = sg;
79 if (!sg)
80 return 0;
81
82 wa->nents = sg_nents_for_len(sg, len);
83 if (wa->nents < 0)
84 return wa->nents;
85
86 wa->bytes_left = len;
87 wa->sg_used = 0;
88
89 if (len == 0)
90 return 0;
91
92 if (dma_dir == DMA_NONE)
93 return 0;
94
95 wa->dma_sg = sg;
96 wa->dma_sg_head = sg;
97 wa->dma_dev = dev;
98 wa->dma_dir = dma_dir;
99 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
100 if (!wa->dma_count)
101 return -ENOMEM;
102
103 return 0;
104}
105
106static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
107{
108 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
109 unsigned int sg_combined_len = 0;
110
111 if (!wa->sg)
112 return;
113
114 wa->sg_used += nbytes;
115 wa->bytes_left -= nbytes;
116 if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
117 /* Advance to the next DMA scatterlist entry */
118 wa->dma_sg = sg_next(wa->dma_sg);
119
120 /* In the case that the DMA mapped scatterlist has entries
121 * that have been merged, the non-DMA mapped scatterlist
122 * must be advanced multiple times for each merged entry.
123 * This ensures that the current non-DMA mapped entry
124 * corresponds to the current DMA mapped entry.
125 */
126 do {
127 sg_combined_len += wa->sg->length;
128 wa->sg = sg_next(wa->sg);
129 } while (wa->sg_used > sg_combined_len);
130
131 wa->sg_used = 0;
132 }
133}
134
135static void ccp_dm_free(struct ccp_dm_workarea *wa)
136{
137 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
138 if (wa->address)
139 dma_pool_free(wa->dma_pool, wa->address,
140 wa->dma.address);
141 } else {
142 if (wa->dma.address)
143 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
144 wa->dma.dir);
145 kfree(wa->address);
146 }
147
148 wa->address = NULL;
149 wa->dma.address = 0;
150}
151
152static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
153 struct ccp_cmd_queue *cmd_q,
154 unsigned int len,
155 enum dma_data_direction dir)
156{
157 memset(wa, 0, sizeof(*wa));
158
159 if (!len)
160 return 0;
161
162 wa->dev = cmd_q->ccp->dev;
163 wa->length = len;
164
165 if (len <= CCP_DMAPOOL_MAX_SIZE) {
166 wa->dma_pool = cmd_q->dma_pool;
167
168 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
169 &wa->dma.address);
170 if (!wa->address)
171 return -ENOMEM;
172
173 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
174
175 } else {
176 wa->address = kzalloc(len, GFP_KERNEL);
177 if (!wa->address)
178 return -ENOMEM;
179
180 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
181 dir);
182 if (dma_mapping_error(wa->dev, wa->dma.address))
183 return -ENOMEM;
184
185 wa->dma.length = len;
186 }
187 wa->dma.dir = dir;
188
189 return 0;
190}
191
192static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
193 struct scatterlist *sg, unsigned int sg_offset,
194 unsigned int len)
195{
196 WARN_ON(!wa->address);
197
198 if (len > (wa->length - wa_offset))
199 return -EINVAL;
200
201 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
202 0);
203 return 0;
204}
205
206static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
207 struct scatterlist *sg, unsigned int sg_offset,
208 unsigned int len)
209{
210 WARN_ON(!wa->address);
211
212 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
213 1);
214}
215
216static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
217 unsigned int wa_offset,
218 struct scatterlist *sg,
219 unsigned int sg_offset,
220 unsigned int len)
221{
222 u8 *p, *q;
223 int rc;
224
225 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
226 if (rc)
227 return rc;
228
229 p = wa->address + wa_offset;
230 q = p + len - 1;
231 while (p < q) {
232 *p = *p ^ *q;
233 *q = *p ^ *q;
234 *p = *p ^ *q;
235 p++;
236 q--;
237 }
238 return 0;
239}
240
241static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
242 unsigned int wa_offset,
243 struct scatterlist *sg,
244 unsigned int sg_offset,
245 unsigned int len)
246{
247 u8 *p, *q;
248
249 p = wa->address + wa_offset;
250 q = p + len - 1;
251 while (p < q) {
252 *p = *p ^ *q;
253 *q = *p ^ *q;
254 *p = *p ^ *q;
255 p++;
256 q--;
257 }
258
259 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
260}
261
262static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
263{
264 ccp_dm_free(&data->dm_wa);
265 ccp_sg_free(&data->sg_wa);
266}
267
268static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
269 struct scatterlist *sg, u64 sg_len,
270 unsigned int dm_len,
271 enum dma_data_direction dir)
272{
273 int ret;
274
275 memset(data, 0, sizeof(*data));
276
277 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
278 dir);
279 if (ret)
280 goto e_err;
281
282 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
283 if (ret)
284 goto e_err;
285
286 return 0;
287
288e_err:
289 ccp_free_data(data, cmd_q);
290
291 return ret;
292}
293
294static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
295{
296 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
297 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
298 unsigned int buf_count, nbytes;
299
300 /* Clear the buffer if setting it */
301 if (!from)
302 memset(dm_wa->address, 0, dm_wa->length);
303
304 if (!sg_wa->sg)
305 return 0;
306
307 /* Perform the copy operation
308 * nbytes will always be <= UINT_MAX because dm_wa->length is
309 * an unsigned int
310 */
311 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
312 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
313 nbytes, from);
314
315 /* Update the structures and generate the count */
316 buf_count = 0;
317 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
318 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
319 dm_wa->length - buf_count);
320 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
321
322 buf_count += nbytes;
323 ccp_update_sg_workarea(sg_wa, nbytes);
324 }
325
326 return buf_count;
327}
328
329static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
330{
331 return ccp_queue_buf(data, 0);
332}
333
334static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
335{
336 return ccp_queue_buf(data, 1);
337}
338
339static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
340 struct ccp_op *op, unsigned int block_size,
341 bool blocksize_op)
342{
343 unsigned int sg_src_len, sg_dst_len, op_len;
344
345 /* The CCP can only DMA from/to one address each per operation. This
346 * requires that we find the smallest DMA area between the source
347 * and destination. The resulting len values will always be <= UINT_MAX
348 * because the dma length is an unsigned int.
349 */
350 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
351 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
352
353 if (dst) {
354 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
355 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
356 op_len = min(sg_src_len, sg_dst_len);
357 } else {
358 op_len = sg_src_len;
359 }
360
361 /* The data operation length will be at least block_size in length
362 * or the smaller of available sg room remaining for the source or
363 * the destination
364 */
365 op_len = max(op_len, block_size);
366
367 /* Unless we have to buffer data, there's no reason to wait */
368 op->soc = 0;
369
370 if (sg_src_len < block_size) {
371 /* Not enough data in the sg element, so it
372 * needs to be buffered into a blocksize chunk
373 */
374 int cp_len = ccp_fill_queue_buf(src);
375
376 op->soc = 1;
377 op->src.u.dma.address = src->dm_wa.dma.address;
378 op->src.u.dma.offset = 0;
379 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
380 } else {
381 /* Enough data in the sg element, but we need to
382 * adjust for any previously copied data
383 */
384 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
385 op->src.u.dma.offset = src->sg_wa.sg_used;
386 op->src.u.dma.length = op_len & ~(block_size - 1);
387
388 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
389 }
390
391 if (dst) {
392 if (sg_dst_len < block_size) {
393 /* Not enough room in the sg element or we're on the
394 * last piece of data (when using padding), so the
395 * output needs to be buffered into a blocksize chunk
396 */
397 op->soc = 1;
398 op->dst.u.dma.address = dst->dm_wa.dma.address;
399 op->dst.u.dma.offset = 0;
400 op->dst.u.dma.length = op->src.u.dma.length;
401 } else {
402 /* Enough room in the sg element, but we need to
403 * adjust for any previously used area
404 */
405 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
406 op->dst.u.dma.offset = dst->sg_wa.sg_used;
407 op->dst.u.dma.length = op->src.u.dma.length;
408 }
409 }
410}
411
412static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
413 struct ccp_op *op)
414{
415 op->init = 0;
416
417 if (dst) {
418 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
419 ccp_empty_queue_buf(dst);
420 else
421 ccp_update_sg_workarea(&dst->sg_wa,
422 op->dst.u.dma.length);
423 }
424}
425
426static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
427 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
428 u32 byte_swap, bool from)
429{
430 struct ccp_op op;
431
432 memset(&op, 0, sizeof(op));
433
434 op.cmd_q = cmd_q;
435 op.jobid = jobid;
436 op.eom = 1;
437
438 if (from) {
439 op.soc = 1;
440 op.src.type = CCP_MEMTYPE_SB;
441 op.src.u.sb = sb;
442 op.dst.type = CCP_MEMTYPE_SYSTEM;
443 op.dst.u.dma.address = wa->dma.address;
444 op.dst.u.dma.length = wa->length;
445 } else {
446 op.src.type = CCP_MEMTYPE_SYSTEM;
447 op.src.u.dma.address = wa->dma.address;
448 op.src.u.dma.length = wa->length;
449 op.dst.type = CCP_MEMTYPE_SB;
450 op.dst.u.sb = sb;
451 }
452
453 op.u.passthru.byte_swap = byte_swap;
454
455 return cmd_q->ccp->vdata->perform->passthru(&op);
456}
457
458static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
459 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
460 u32 byte_swap)
461{
462 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
463}
464
465static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
466 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
467 u32 byte_swap)
468{
469 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
470}
471
472static noinline_for_stack int
473ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
474{
475 struct ccp_aes_engine *aes = &cmd->u.aes;
476 struct ccp_dm_workarea key, ctx;
477 struct ccp_data src;
478 struct ccp_op op;
479 unsigned int dm_offset;
480 int ret;
481
482 if (!((aes->key_len == AES_KEYSIZE_128) ||
483 (aes->key_len == AES_KEYSIZE_192) ||
484 (aes->key_len == AES_KEYSIZE_256)))
485 return -EINVAL;
486
487 if (aes->src_len & (AES_BLOCK_SIZE - 1))
488 return -EINVAL;
489
490 if (aes->iv_len != AES_BLOCK_SIZE)
491 return -EINVAL;
492
493 if (!aes->key || !aes->iv || !aes->src)
494 return -EINVAL;
495
496 if (aes->cmac_final) {
497 if (aes->cmac_key_len != AES_BLOCK_SIZE)
498 return -EINVAL;
499
500 if (!aes->cmac_key)
501 return -EINVAL;
502 }
503
504 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
505 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
506
507 ret = -EIO;
508 memset(&op, 0, sizeof(op));
509 op.cmd_q = cmd_q;
510 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
511 op.sb_key = cmd_q->sb_key;
512 op.sb_ctx = cmd_q->sb_ctx;
513 op.init = 1;
514 op.u.aes.type = aes->type;
515 op.u.aes.mode = aes->mode;
516 op.u.aes.action = aes->action;
517
518 /* All supported key sizes fit in a single (32-byte) SB entry
519 * and must be in little endian format. Use the 256-bit byte
520 * swap passthru option to convert from big endian to little
521 * endian.
522 */
523 ret = ccp_init_dm_workarea(&key, cmd_q,
524 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
525 DMA_TO_DEVICE);
526 if (ret)
527 return ret;
528
529 dm_offset = CCP_SB_BYTES - aes->key_len;
530 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
531 if (ret)
532 goto e_key;
533 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
534 CCP_PASSTHRU_BYTESWAP_256BIT);
535 if (ret) {
536 cmd->engine_error = cmd_q->cmd_error;
537 goto e_key;
538 }
539
540 /* The AES context fits in a single (32-byte) SB entry and
541 * must be in little endian format. Use the 256-bit byte swap
542 * passthru option to convert from big endian to little endian.
543 */
544 ret = ccp_init_dm_workarea(&ctx, cmd_q,
545 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
546 DMA_BIDIRECTIONAL);
547 if (ret)
548 goto e_key;
549
550 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
551 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
552 if (ret)
553 goto e_ctx;
554 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
555 CCP_PASSTHRU_BYTESWAP_256BIT);
556 if (ret) {
557 cmd->engine_error = cmd_q->cmd_error;
558 goto e_ctx;
559 }
560
561 /* Send data to the CCP AES engine */
562 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
563 AES_BLOCK_SIZE, DMA_TO_DEVICE);
564 if (ret)
565 goto e_ctx;
566
567 while (src.sg_wa.bytes_left) {
568 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
569 if (aes->cmac_final && !src.sg_wa.bytes_left) {
570 op.eom = 1;
571
572 /* Push the K1/K2 key to the CCP now */
573 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
574 op.sb_ctx,
575 CCP_PASSTHRU_BYTESWAP_256BIT);
576 if (ret) {
577 cmd->engine_error = cmd_q->cmd_error;
578 goto e_src;
579 }
580
581 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
582 aes->cmac_key_len);
583 if (ret)
584 goto e_src;
585 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
586 CCP_PASSTHRU_BYTESWAP_256BIT);
587 if (ret) {
588 cmd->engine_error = cmd_q->cmd_error;
589 goto e_src;
590 }
591 }
592
593 ret = cmd_q->ccp->vdata->perform->aes(&op);
594 if (ret) {
595 cmd->engine_error = cmd_q->cmd_error;
596 goto e_src;
597 }
598
599 ccp_process_data(&src, NULL, &op);
600 }
601
602 /* Retrieve the AES context - convert from LE to BE using
603 * 32-byte (256-bit) byteswapping
604 */
605 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
606 CCP_PASSTHRU_BYTESWAP_256BIT);
607 if (ret) {
608 cmd->engine_error = cmd_q->cmd_error;
609 goto e_src;
610 }
611
612 /* ...but we only need AES_BLOCK_SIZE bytes */
613 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
614 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
615
616e_src:
617 ccp_free_data(&src, cmd_q);
618
619e_ctx:
620 ccp_dm_free(&ctx);
621
622e_key:
623 ccp_dm_free(&key);
624
625 return ret;
626}
627
628static noinline_for_stack int
629ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
630{
631 struct ccp_aes_engine *aes = &cmd->u.aes;
632 struct ccp_dm_workarea key, ctx, final_wa, tag;
633 struct ccp_data src, dst;
634 struct ccp_data aad;
635 struct ccp_op op;
636 unsigned int dm_offset;
637 unsigned int authsize;
638 unsigned int jobid;
639 unsigned int ilen;
640 bool in_place = true; /* Default value */
641 __be64 *final;
642 int ret;
643
644 struct scatterlist *p_inp, sg_inp[2];
645 struct scatterlist *p_tag, sg_tag[2];
646 struct scatterlist *p_outp, sg_outp[2];
647 struct scatterlist *p_aad;
648
649 if (!aes->iv)
650 return -EINVAL;
651
652 if (!((aes->key_len == AES_KEYSIZE_128) ||
653 (aes->key_len == AES_KEYSIZE_192) ||
654 (aes->key_len == AES_KEYSIZE_256)))
655 return -EINVAL;
656
657 if (!aes->key) /* Gotta have a key SGL */
658 return -EINVAL;
659
660 /* Zero defaults to 16 bytes, the maximum size */
661 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
662 switch (authsize) {
663 case 16:
664 case 15:
665 case 14:
666 case 13:
667 case 12:
668 case 8:
669 case 4:
670 break;
671 default:
672 return -EINVAL;
673 }
674
675 /* First, decompose the source buffer into AAD & PT,
676 * and the destination buffer into AAD, CT & tag, or
677 * the input into CT & tag.
678 * It is expected that the input and output SGs will
679 * be valid, even if the AAD and input lengths are 0.
680 */
681 p_aad = aes->src;
682 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
683 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
684 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
685 ilen = aes->src_len;
686 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
687 } else {
688 /* Input length for decryption includes tag */
689 ilen = aes->src_len - authsize;
690 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
691 }
692
693 jobid = CCP_NEW_JOBID(cmd_q->ccp);
694
695 memset(&op, 0, sizeof(op));
696 op.cmd_q = cmd_q;
697 op.jobid = jobid;
698 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
699 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
700 op.init = 1;
701 op.u.aes.type = aes->type;
702
703 /* Copy the key to the LSB */
704 ret = ccp_init_dm_workarea(&key, cmd_q,
705 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
706 DMA_TO_DEVICE);
707 if (ret)
708 return ret;
709
710 dm_offset = CCP_SB_BYTES - aes->key_len;
711 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
712 if (ret)
713 goto e_key;
714 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
715 CCP_PASSTHRU_BYTESWAP_256BIT);
716 if (ret) {
717 cmd->engine_error = cmd_q->cmd_error;
718 goto e_key;
719 }
720
721 /* Copy the context (IV) to the LSB.
722 * There is an assumption here that the IV is 96 bits in length, plus
723 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
724 */
725 ret = ccp_init_dm_workarea(&ctx, cmd_q,
726 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
727 DMA_BIDIRECTIONAL);
728 if (ret)
729 goto e_key;
730
731 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
732 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
733 if (ret)
734 goto e_ctx;
735
736 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
737 CCP_PASSTHRU_BYTESWAP_256BIT);
738 if (ret) {
739 cmd->engine_error = cmd_q->cmd_error;
740 goto e_ctx;
741 }
742
743 op.init = 1;
744 if (aes->aad_len > 0) {
745 /* Step 1: Run a GHASH over the Additional Authenticated Data */
746 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
747 AES_BLOCK_SIZE,
748 DMA_TO_DEVICE);
749 if (ret)
750 goto e_ctx;
751
752 op.u.aes.mode = CCP_AES_MODE_GHASH;
753 op.u.aes.action = CCP_AES_GHASHAAD;
754
755 while (aad.sg_wa.bytes_left) {
756 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
757
758 ret = cmd_q->ccp->vdata->perform->aes(&op);
759 if (ret) {
760 cmd->engine_error = cmd_q->cmd_error;
761 goto e_aad;
762 }
763
764 ccp_process_data(&aad, NULL, &op);
765 op.init = 0;
766 }
767 }
768
769 op.u.aes.mode = CCP_AES_MODE_GCTR;
770 op.u.aes.action = aes->action;
771
772 if (ilen > 0) {
773 /* Step 2: Run a GCTR over the plaintext */
774 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
775
776 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
777 AES_BLOCK_SIZE,
778 in_place ? DMA_BIDIRECTIONAL
779 : DMA_TO_DEVICE);
780 if (ret)
781 goto e_aad;
782
783 if (in_place) {
784 dst = src;
785 } else {
786 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
787 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
788 if (ret)
789 goto e_src;
790 }
791
792 op.soc = 0;
793 op.eom = 0;
794 op.init = 1;
795 while (src.sg_wa.bytes_left) {
796 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
797 if (!src.sg_wa.bytes_left) {
798 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
799
800 if (nbytes) {
801 op.eom = 1;
802 op.u.aes.size = (nbytes * 8) - 1;
803 }
804 }
805
806 ret = cmd_q->ccp->vdata->perform->aes(&op);
807 if (ret) {
808 cmd->engine_error = cmd_q->cmd_error;
809 goto e_dst;
810 }
811
812 ccp_process_data(&src, &dst, &op);
813 op.init = 0;
814 }
815 }
816
817 /* Step 3: Update the IV portion of the context with the original IV */
818 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
819 CCP_PASSTHRU_BYTESWAP_256BIT);
820 if (ret) {
821 cmd->engine_error = cmd_q->cmd_error;
822 goto e_dst;
823 }
824
825 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
826 if (ret)
827 goto e_dst;
828
829 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
830 CCP_PASSTHRU_BYTESWAP_256BIT);
831 if (ret) {
832 cmd->engine_error = cmd_q->cmd_error;
833 goto e_dst;
834 }
835
836 /* Step 4: Concatenate the lengths of the AAD and source, and
837 * hash that 16 byte buffer.
838 */
839 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
840 DMA_BIDIRECTIONAL);
841 if (ret)
842 goto e_dst;
843 final = (__be64 *)final_wa.address;
844 final[0] = cpu_to_be64(aes->aad_len * 8);
845 final[1] = cpu_to_be64(ilen * 8);
846
847 memset(&op, 0, sizeof(op));
848 op.cmd_q = cmd_q;
849 op.jobid = jobid;
850 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
851 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
852 op.init = 1;
853 op.u.aes.type = aes->type;
854 op.u.aes.mode = CCP_AES_MODE_GHASH;
855 op.u.aes.action = CCP_AES_GHASHFINAL;
856 op.src.type = CCP_MEMTYPE_SYSTEM;
857 op.src.u.dma.address = final_wa.dma.address;
858 op.src.u.dma.length = AES_BLOCK_SIZE;
859 op.dst.type = CCP_MEMTYPE_SYSTEM;
860 op.dst.u.dma.address = final_wa.dma.address;
861 op.dst.u.dma.length = AES_BLOCK_SIZE;
862 op.eom = 1;
863 op.u.aes.size = 0;
864 ret = cmd_q->ccp->vdata->perform->aes(&op);
865 if (ret)
866 goto e_final_wa;
867
868 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
869 /* Put the ciphered tag after the ciphertext. */
870 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
871 } else {
872 /* Does this ciphered tag match the input? */
873 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
874 DMA_BIDIRECTIONAL);
875 if (ret)
876 goto e_final_wa;
877 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
878 if (ret) {
879 ccp_dm_free(&tag);
880 goto e_final_wa;
881 }
882
883 ret = crypto_memneq(tag.address, final_wa.address,
884 authsize) ? -EBADMSG : 0;
885 ccp_dm_free(&tag);
886 }
887
888e_final_wa:
889 ccp_dm_free(&final_wa);
890
891e_dst:
892 if (ilen > 0 && !in_place)
893 ccp_free_data(&dst, cmd_q);
894
895e_src:
896 if (ilen > 0)
897 ccp_free_data(&src, cmd_q);
898
899e_aad:
900 if (aes->aad_len)
901 ccp_free_data(&aad, cmd_q);
902
903e_ctx:
904 ccp_dm_free(&ctx);
905
906e_key:
907 ccp_dm_free(&key);
908
909 return ret;
910}
911
912static noinline_for_stack int
913ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
914{
915 struct ccp_aes_engine *aes = &cmd->u.aes;
916 struct ccp_dm_workarea key, ctx;
917 struct ccp_data src, dst;
918 struct ccp_op op;
919 unsigned int dm_offset;
920 bool in_place = false;
921 int ret;
922
923 if (!((aes->key_len == AES_KEYSIZE_128) ||
924 (aes->key_len == AES_KEYSIZE_192) ||
925 (aes->key_len == AES_KEYSIZE_256)))
926 return -EINVAL;
927
928 if (((aes->mode == CCP_AES_MODE_ECB) ||
929 (aes->mode == CCP_AES_MODE_CBC)) &&
930 (aes->src_len & (AES_BLOCK_SIZE - 1)))
931 return -EINVAL;
932
933 if (!aes->key || !aes->src || !aes->dst)
934 return -EINVAL;
935
936 if (aes->mode != CCP_AES_MODE_ECB) {
937 if (aes->iv_len != AES_BLOCK_SIZE)
938 return -EINVAL;
939
940 if (!aes->iv)
941 return -EINVAL;
942 }
943
944 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
945 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
946
947 ret = -EIO;
948 memset(&op, 0, sizeof(op));
949 op.cmd_q = cmd_q;
950 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
951 op.sb_key = cmd_q->sb_key;
952 op.sb_ctx = cmd_q->sb_ctx;
953 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
954 op.u.aes.type = aes->type;
955 op.u.aes.mode = aes->mode;
956 op.u.aes.action = aes->action;
957
958 /* All supported key sizes fit in a single (32-byte) SB entry
959 * and must be in little endian format. Use the 256-bit byte
960 * swap passthru option to convert from big endian to little
961 * endian.
962 */
963 ret = ccp_init_dm_workarea(&key, cmd_q,
964 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
965 DMA_TO_DEVICE);
966 if (ret)
967 return ret;
968
969 dm_offset = CCP_SB_BYTES - aes->key_len;
970 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
971 if (ret)
972 goto e_key;
973 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
974 CCP_PASSTHRU_BYTESWAP_256BIT);
975 if (ret) {
976 cmd->engine_error = cmd_q->cmd_error;
977 goto e_key;
978 }
979
980 /* The AES context fits in a single (32-byte) SB entry and
981 * must be in little endian format. Use the 256-bit byte swap
982 * passthru option to convert from big endian to little endian.
983 */
984 ret = ccp_init_dm_workarea(&ctx, cmd_q,
985 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
986 DMA_BIDIRECTIONAL);
987 if (ret)
988 goto e_key;
989
990 if (aes->mode != CCP_AES_MODE_ECB) {
991 /* Load the AES context - convert to LE */
992 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
993 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
994 if (ret)
995 goto e_ctx;
996 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
997 CCP_PASSTHRU_BYTESWAP_256BIT);
998 if (ret) {
999 cmd->engine_error = cmd_q->cmd_error;
1000 goto e_ctx;
1001 }
1002 }
1003 switch (aes->mode) {
1004 case CCP_AES_MODE_CFB: /* CFB128 only */
1005 case CCP_AES_MODE_CTR:
1006 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1007 break;
1008 default:
1009 op.u.aes.size = 0;
1010 }
1011
1012 /* Prepare the input and output data workareas. For in-place
1013 * operations we need to set the dma direction to BIDIRECTIONAL
1014 * and copy the src workarea to the dst workarea.
1015 */
1016 if (sg_virt(aes->src) == sg_virt(aes->dst))
1017 in_place = true;
1018
1019 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1020 AES_BLOCK_SIZE,
1021 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1022 if (ret)
1023 goto e_ctx;
1024
1025 if (in_place) {
1026 dst = src;
1027 } else {
1028 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1029 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1030 if (ret)
1031 goto e_src;
1032 }
1033
1034 /* Send data to the CCP AES engine */
1035 while (src.sg_wa.bytes_left) {
1036 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1037 if (!src.sg_wa.bytes_left) {
1038 op.eom = 1;
1039
1040 /* Since we don't retrieve the AES context in ECB
1041 * mode we have to wait for the operation to complete
1042 * on the last piece of data
1043 */
1044 if (aes->mode == CCP_AES_MODE_ECB)
1045 op.soc = 1;
1046 }
1047
1048 ret = cmd_q->ccp->vdata->perform->aes(&op);
1049 if (ret) {
1050 cmd->engine_error = cmd_q->cmd_error;
1051 goto e_dst;
1052 }
1053
1054 ccp_process_data(&src, &dst, &op);
1055 }
1056
1057 if (aes->mode != CCP_AES_MODE_ECB) {
1058 /* Retrieve the AES context - convert from LE to BE using
1059 * 32-byte (256-bit) byteswapping
1060 */
1061 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1062 CCP_PASSTHRU_BYTESWAP_256BIT);
1063 if (ret) {
1064 cmd->engine_error = cmd_q->cmd_error;
1065 goto e_dst;
1066 }
1067
1068 /* ...but we only need AES_BLOCK_SIZE bytes */
1069 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1070 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1071 }
1072
1073e_dst:
1074 if (!in_place)
1075 ccp_free_data(&dst, cmd_q);
1076
1077e_src:
1078 ccp_free_data(&src, cmd_q);
1079
1080e_ctx:
1081 ccp_dm_free(&ctx);
1082
1083e_key:
1084 ccp_dm_free(&key);
1085
1086 return ret;
1087}
1088
1089static noinline_for_stack int
1090ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1091{
1092 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1093 struct ccp_dm_workarea key, ctx;
1094 struct ccp_data src, dst;
1095 struct ccp_op op;
1096 unsigned int unit_size, dm_offset;
1097 bool in_place = false;
1098 unsigned int sb_count;
1099 enum ccp_aes_type aestype;
1100 int ret;
1101
1102 switch (xts->unit_size) {
1103 case CCP_XTS_AES_UNIT_SIZE_16:
1104 unit_size = 16;
1105 break;
1106 case CCP_XTS_AES_UNIT_SIZE_512:
1107 unit_size = 512;
1108 break;
1109 case CCP_XTS_AES_UNIT_SIZE_1024:
1110 unit_size = 1024;
1111 break;
1112 case CCP_XTS_AES_UNIT_SIZE_2048:
1113 unit_size = 2048;
1114 break;
1115 case CCP_XTS_AES_UNIT_SIZE_4096:
1116 unit_size = 4096;
1117 break;
1118
1119 default:
1120 return -EINVAL;
1121 }
1122
1123 if (xts->key_len == AES_KEYSIZE_128)
1124 aestype = CCP_AES_TYPE_128;
1125 else if (xts->key_len == AES_KEYSIZE_256)
1126 aestype = CCP_AES_TYPE_256;
1127 else
1128 return -EINVAL;
1129
1130 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1131 return -EINVAL;
1132
1133 if (xts->iv_len != AES_BLOCK_SIZE)
1134 return -EINVAL;
1135
1136 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1137 return -EINVAL;
1138
1139 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1140 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1141
1142 ret = -EIO;
1143 memset(&op, 0, sizeof(op));
1144 op.cmd_q = cmd_q;
1145 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1146 op.sb_key = cmd_q->sb_key;
1147 op.sb_ctx = cmd_q->sb_ctx;
1148 op.init = 1;
1149 op.u.xts.type = aestype;
1150 op.u.xts.action = xts->action;
1151 op.u.xts.unit_size = xts->unit_size;
1152
1153 /* A version 3 device only supports 128-bit keys, which fits into a
1154 * single SB entry. A version 5 device uses a 512-bit vector, so two
1155 * SB entries.
1156 */
1157 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1158 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1159 else
1160 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1161 ret = ccp_init_dm_workarea(&key, cmd_q,
1162 sb_count * CCP_SB_BYTES,
1163 DMA_TO_DEVICE);
1164 if (ret)
1165 return ret;
1166
1167 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1168 /* All supported key sizes must be in little endian format.
1169 * Use the 256-bit byte swap passthru option to convert from
1170 * big endian to little endian.
1171 */
1172 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1173 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1174 if (ret)
1175 goto e_key;
1176 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1177 if (ret)
1178 goto e_key;
1179 } else {
1180 /* Version 5 CCPs use a 512-bit space for the key: each portion
1181 * occupies 256 bits, or one entire slot, and is zero-padded.
1182 */
1183 unsigned int pad;
1184
1185 dm_offset = CCP_SB_BYTES;
1186 pad = dm_offset - xts->key_len;
1187 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1188 if (ret)
1189 goto e_key;
1190 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1191 xts->key_len, xts->key_len);
1192 if (ret)
1193 goto e_key;
1194 }
1195 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1196 CCP_PASSTHRU_BYTESWAP_256BIT);
1197 if (ret) {
1198 cmd->engine_error = cmd_q->cmd_error;
1199 goto e_key;
1200 }
1201
1202 /* The AES context fits in a single (32-byte) SB entry and
1203 * for XTS is already in little endian format so no byte swapping
1204 * is needed.
1205 */
1206 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1207 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1208 DMA_BIDIRECTIONAL);
1209 if (ret)
1210 goto e_key;
1211
1212 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1213 if (ret)
1214 goto e_ctx;
1215 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1216 CCP_PASSTHRU_BYTESWAP_NOOP);
1217 if (ret) {
1218 cmd->engine_error = cmd_q->cmd_error;
1219 goto e_ctx;
1220 }
1221
1222 /* Prepare the input and output data workareas. For in-place
1223 * operations we need to set the dma direction to BIDIRECTIONAL
1224 * and copy the src workarea to the dst workarea.
1225 */
1226 if (sg_virt(xts->src) == sg_virt(xts->dst))
1227 in_place = true;
1228
1229 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1230 unit_size,
1231 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1232 if (ret)
1233 goto e_ctx;
1234
1235 if (in_place) {
1236 dst = src;
1237 } else {
1238 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1239 unit_size, DMA_FROM_DEVICE);
1240 if (ret)
1241 goto e_src;
1242 }
1243
1244 /* Send data to the CCP AES engine */
1245 while (src.sg_wa.bytes_left) {
1246 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1247 if (!src.sg_wa.bytes_left)
1248 op.eom = 1;
1249
1250 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1251 if (ret) {
1252 cmd->engine_error = cmd_q->cmd_error;
1253 goto e_dst;
1254 }
1255
1256 ccp_process_data(&src, &dst, &op);
1257 }
1258
1259 /* Retrieve the AES context - convert from LE to BE using
1260 * 32-byte (256-bit) byteswapping
1261 */
1262 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1263 CCP_PASSTHRU_BYTESWAP_256BIT);
1264 if (ret) {
1265 cmd->engine_error = cmd_q->cmd_error;
1266 goto e_dst;
1267 }
1268
1269 /* ...but we only need AES_BLOCK_SIZE bytes */
1270 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1271 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1272
1273e_dst:
1274 if (!in_place)
1275 ccp_free_data(&dst, cmd_q);
1276
1277e_src:
1278 ccp_free_data(&src, cmd_q);
1279
1280e_ctx:
1281 ccp_dm_free(&ctx);
1282
1283e_key:
1284 ccp_dm_free(&key);
1285
1286 return ret;
1287}
1288
1289static noinline_for_stack int
1290ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1291{
1292 struct ccp_des3_engine *des3 = &cmd->u.des3;
1293
1294 struct ccp_dm_workarea key, ctx;
1295 struct ccp_data src, dst;
1296 struct ccp_op op;
1297 unsigned int dm_offset;
1298 unsigned int len_singlekey;
1299 bool in_place = false;
1300 int ret;
1301
1302 /* Error checks */
1303 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1304 return -EINVAL;
1305
1306 if (!cmd_q->ccp->vdata->perform->des3)
1307 return -EINVAL;
1308
1309 if (des3->key_len != DES3_EDE_KEY_SIZE)
1310 return -EINVAL;
1311
1312 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1313 (des3->mode == CCP_DES3_MODE_CBC)) &&
1314 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1315 return -EINVAL;
1316
1317 if (!des3->key || !des3->src || !des3->dst)
1318 return -EINVAL;
1319
1320 if (des3->mode != CCP_DES3_MODE_ECB) {
1321 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1322 return -EINVAL;
1323
1324 if (!des3->iv)
1325 return -EINVAL;
1326 }
1327
1328 /* Zero out all the fields of the command desc */
1329 memset(&op, 0, sizeof(op));
1330
1331 /* Set up the Function field */
1332 op.cmd_q = cmd_q;
1333 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1334 op.sb_key = cmd_q->sb_key;
1335
1336 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1337 op.u.des3.type = des3->type;
1338 op.u.des3.mode = des3->mode;
1339 op.u.des3.action = des3->action;
1340
1341 /*
1342 * All supported key sizes fit in a single (32-byte) KSB entry and
1343 * (like AES) must be in little endian format. Use the 256-bit byte
1344 * swap passthru option to convert from big endian to little endian.
1345 */
1346 ret = ccp_init_dm_workarea(&key, cmd_q,
1347 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1348 DMA_TO_DEVICE);
1349 if (ret)
1350 return ret;
1351
1352 /*
1353 * The contents of the key triplet are in the reverse order of what
1354 * is required by the engine. Copy the 3 pieces individually to put
1355 * them where they belong.
1356 */
1357 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1358
1359 len_singlekey = des3->key_len / 3;
1360 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1361 des3->key, 0, len_singlekey);
1362 if (ret)
1363 goto e_key;
1364 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1365 des3->key, len_singlekey, len_singlekey);
1366 if (ret)
1367 goto e_key;
1368 ret = ccp_set_dm_area(&key, dm_offset,
1369 des3->key, 2 * len_singlekey, len_singlekey);
1370 if (ret)
1371 goto e_key;
1372
1373 /* Copy the key to the SB */
1374 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1375 CCP_PASSTHRU_BYTESWAP_256BIT);
1376 if (ret) {
1377 cmd->engine_error = cmd_q->cmd_error;
1378 goto e_key;
1379 }
1380
1381 /*
1382 * The DES3 context fits in a single (32-byte) KSB entry and
1383 * must be in little endian format. Use the 256-bit byte swap
1384 * passthru option to convert from big endian to little endian.
1385 */
1386 if (des3->mode != CCP_DES3_MODE_ECB) {
1387 op.sb_ctx = cmd_q->sb_ctx;
1388
1389 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1390 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1391 DMA_BIDIRECTIONAL);
1392 if (ret)
1393 goto e_key;
1394
1395 /* Load the context into the LSB */
1396 dm_offset = CCP_SB_BYTES - des3->iv_len;
1397 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1398 des3->iv_len);
1399 if (ret)
1400 goto e_ctx;
1401
1402 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1403 CCP_PASSTHRU_BYTESWAP_256BIT);
1404 if (ret) {
1405 cmd->engine_error = cmd_q->cmd_error;
1406 goto e_ctx;
1407 }
1408 }
1409
1410 /*
1411 * Prepare the input and output data workareas. For in-place
1412 * operations we need to set the dma direction to BIDIRECTIONAL
1413 * and copy the src workarea to the dst workarea.
1414 */
1415 if (sg_virt(des3->src) == sg_virt(des3->dst))
1416 in_place = true;
1417
1418 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1419 DES3_EDE_BLOCK_SIZE,
1420 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1421 if (ret)
1422 goto e_ctx;
1423
1424 if (in_place)
1425 dst = src;
1426 else {
1427 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1428 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1429 if (ret)
1430 goto e_src;
1431 }
1432
1433 /* Send data to the CCP DES3 engine */
1434 while (src.sg_wa.bytes_left) {
1435 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1436 if (!src.sg_wa.bytes_left) {
1437 op.eom = 1;
1438
1439 /* Since we don't retrieve the context in ECB mode
1440 * we have to wait for the operation to complete
1441 * on the last piece of data
1442 */
1443 op.soc = 0;
1444 }
1445
1446 ret = cmd_q->ccp->vdata->perform->des3(&op);
1447 if (ret) {
1448 cmd->engine_error = cmd_q->cmd_error;
1449 goto e_dst;
1450 }
1451
1452 ccp_process_data(&src, &dst, &op);
1453 }
1454
1455 if (des3->mode != CCP_DES3_MODE_ECB) {
1456 /* Retrieve the context and make BE */
1457 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1458 CCP_PASSTHRU_BYTESWAP_256BIT);
1459 if (ret) {
1460 cmd->engine_error = cmd_q->cmd_error;
1461 goto e_dst;
1462 }
1463
1464 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1465 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1466 DES3_EDE_BLOCK_SIZE);
1467 }
1468e_dst:
1469 if (!in_place)
1470 ccp_free_data(&dst, cmd_q);
1471
1472e_src:
1473 ccp_free_data(&src, cmd_q);
1474
1475e_ctx:
1476 if (des3->mode != CCP_DES3_MODE_ECB)
1477 ccp_dm_free(&ctx);
1478
1479e_key:
1480 ccp_dm_free(&key);
1481
1482 return ret;
1483}
1484
1485static noinline_for_stack int
1486ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1487{
1488 struct ccp_sha_engine *sha = &cmd->u.sha;
1489 struct ccp_dm_workarea ctx;
1490 struct ccp_data src;
1491 struct ccp_op op;
1492 unsigned int ioffset, ooffset;
1493 unsigned int digest_size;
1494 int sb_count;
1495 const void *init;
1496 u64 block_size;
1497 int ctx_size;
1498 int ret;
1499
1500 switch (sha->type) {
1501 case CCP_SHA_TYPE_1:
1502 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1503 return -EINVAL;
1504 block_size = SHA1_BLOCK_SIZE;
1505 break;
1506 case CCP_SHA_TYPE_224:
1507 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1508 return -EINVAL;
1509 block_size = SHA224_BLOCK_SIZE;
1510 break;
1511 case CCP_SHA_TYPE_256:
1512 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1513 return -EINVAL;
1514 block_size = SHA256_BLOCK_SIZE;
1515 break;
1516 case CCP_SHA_TYPE_384:
1517 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1518 || sha->ctx_len < SHA384_DIGEST_SIZE)
1519 return -EINVAL;
1520 block_size = SHA384_BLOCK_SIZE;
1521 break;
1522 case CCP_SHA_TYPE_512:
1523 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1524 || sha->ctx_len < SHA512_DIGEST_SIZE)
1525 return -EINVAL;
1526 block_size = SHA512_BLOCK_SIZE;
1527 break;
1528 default:
1529 return -EINVAL;
1530 }
1531
1532 if (!sha->ctx)
1533 return -EINVAL;
1534
1535 if (!sha->final && (sha->src_len & (block_size - 1)))
1536 return -EINVAL;
1537
1538 /* The version 3 device can't handle zero-length input */
1539 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1540
1541 if (!sha->src_len) {
1542 unsigned int digest_len;
1543 const u8 *sha_zero;
1544
1545 /* Not final, just return */
1546 if (!sha->final)
1547 return 0;
1548
1549 /* CCP can't do a zero length sha operation so the
1550 * caller must buffer the data.
1551 */
1552 if (sha->msg_bits)
1553 return -EINVAL;
1554
1555 /* The CCP cannot perform zero-length sha operations
1556 * so the caller is required to buffer data for the
1557 * final operation. However, a sha operation for a
1558 * message with a total length of zero is valid so
1559 * known values are required to supply the result.
1560 */
1561 switch (sha->type) {
1562 case CCP_SHA_TYPE_1:
1563 sha_zero = sha1_zero_message_hash;
1564 digest_len = SHA1_DIGEST_SIZE;
1565 break;
1566 case CCP_SHA_TYPE_224:
1567 sha_zero = sha224_zero_message_hash;
1568 digest_len = SHA224_DIGEST_SIZE;
1569 break;
1570 case CCP_SHA_TYPE_256:
1571 sha_zero = sha256_zero_message_hash;
1572 digest_len = SHA256_DIGEST_SIZE;
1573 break;
1574 default:
1575 return -EINVAL;
1576 }
1577
1578 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1579 digest_len, 1);
1580
1581 return 0;
1582 }
1583 }
1584
1585 /* Set variables used throughout */
1586 switch (sha->type) {
1587 case CCP_SHA_TYPE_1:
1588 digest_size = SHA1_DIGEST_SIZE;
1589 init = (void *) ccp_sha1_init;
1590 ctx_size = SHA1_DIGEST_SIZE;
1591 sb_count = 1;
1592 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1594 else
1595 ooffset = ioffset = 0;
1596 break;
1597 case CCP_SHA_TYPE_224:
1598 digest_size = SHA224_DIGEST_SIZE;
1599 init = (void *) ccp_sha224_init;
1600 ctx_size = SHA256_DIGEST_SIZE;
1601 sb_count = 1;
1602 ioffset = 0;
1603 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1604 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1605 else
1606 ooffset = 0;
1607 break;
1608 case CCP_SHA_TYPE_256:
1609 digest_size = SHA256_DIGEST_SIZE;
1610 init = (void *) ccp_sha256_init;
1611 ctx_size = SHA256_DIGEST_SIZE;
1612 sb_count = 1;
1613 ooffset = ioffset = 0;
1614 break;
1615 case CCP_SHA_TYPE_384:
1616 digest_size = SHA384_DIGEST_SIZE;
1617 init = (void *) ccp_sha384_init;
1618 ctx_size = SHA512_DIGEST_SIZE;
1619 sb_count = 2;
1620 ioffset = 0;
1621 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1622 break;
1623 case CCP_SHA_TYPE_512:
1624 digest_size = SHA512_DIGEST_SIZE;
1625 init = (void *) ccp_sha512_init;
1626 ctx_size = SHA512_DIGEST_SIZE;
1627 sb_count = 2;
1628 ooffset = ioffset = 0;
1629 break;
1630 default:
1631 ret = -EINVAL;
1632 goto e_data;
1633 }
1634
1635 /* For zero-length plaintext the src pointer is ignored;
1636 * otherwise both parts must be valid
1637 */
1638 if (sha->src_len && !sha->src)
1639 return -EINVAL;
1640
1641 memset(&op, 0, sizeof(op));
1642 op.cmd_q = cmd_q;
1643 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1644 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1645 op.u.sha.type = sha->type;
1646 op.u.sha.msg_bits = sha->msg_bits;
1647
1648 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1649 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1650 * first slot, and the left half in the second. Each portion must then
1651 * be in little endian format: use the 256-bit byte swap option.
1652 */
1653 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1654 DMA_BIDIRECTIONAL);
1655 if (ret)
1656 return ret;
1657 if (sha->first) {
1658 switch (sha->type) {
1659 case CCP_SHA_TYPE_1:
1660 case CCP_SHA_TYPE_224:
1661 case CCP_SHA_TYPE_256:
1662 memcpy(ctx.address + ioffset, init, ctx_size);
1663 break;
1664 case CCP_SHA_TYPE_384:
1665 case CCP_SHA_TYPE_512:
1666 memcpy(ctx.address + ctx_size / 2, init,
1667 ctx_size / 2);
1668 memcpy(ctx.address, init + ctx_size / 2,
1669 ctx_size / 2);
1670 break;
1671 default:
1672 ret = -EINVAL;
1673 goto e_ctx;
1674 }
1675 } else {
1676 /* Restore the context */
1677 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1678 sb_count * CCP_SB_BYTES);
1679 if (ret)
1680 goto e_ctx;
1681 }
1682
1683 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1684 CCP_PASSTHRU_BYTESWAP_256BIT);
1685 if (ret) {
1686 cmd->engine_error = cmd_q->cmd_error;
1687 goto e_ctx;
1688 }
1689
1690 if (sha->src) {
1691 /* Send data to the CCP SHA engine; block_size is set above */
1692 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1693 block_size, DMA_TO_DEVICE);
1694 if (ret)
1695 goto e_ctx;
1696
1697 while (src.sg_wa.bytes_left) {
1698 ccp_prepare_data(&src, NULL, &op, block_size, false);
1699 if (sha->final && !src.sg_wa.bytes_left)
1700 op.eom = 1;
1701
1702 ret = cmd_q->ccp->vdata->perform->sha(&op);
1703 if (ret) {
1704 cmd->engine_error = cmd_q->cmd_error;
1705 goto e_data;
1706 }
1707
1708 ccp_process_data(&src, NULL, &op);
1709 }
1710 } else {
1711 op.eom = 1;
1712 ret = cmd_q->ccp->vdata->perform->sha(&op);
1713 if (ret) {
1714 cmd->engine_error = cmd_q->cmd_error;
1715 goto e_data;
1716 }
1717 }
1718
1719 /* Retrieve the SHA context - convert from LE to BE using
1720 * 32-byte (256-bit) byteswapping to BE
1721 */
1722 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1723 CCP_PASSTHRU_BYTESWAP_256BIT);
1724 if (ret) {
1725 cmd->engine_error = cmd_q->cmd_error;
1726 goto e_data;
1727 }
1728
1729 if (sha->final) {
1730 /* Finishing up, so get the digest */
1731 switch (sha->type) {
1732 case CCP_SHA_TYPE_1:
1733 case CCP_SHA_TYPE_224:
1734 case CCP_SHA_TYPE_256:
1735 ccp_get_dm_area(&ctx, ooffset,
1736 sha->ctx, 0,
1737 digest_size);
1738 break;
1739 case CCP_SHA_TYPE_384:
1740 case CCP_SHA_TYPE_512:
1741 ccp_get_dm_area(&ctx, 0,
1742 sha->ctx, LSB_ITEM_SIZE - ooffset,
1743 LSB_ITEM_SIZE);
1744 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1745 sha->ctx, 0,
1746 LSB_ITEM_SIZE - ooffset);
1747 break;
1748 default:
1749 ret = -EINVAL;
1750 goto e_data;
1751 }
1752 } else {
1753 /* Stash the context */
1754 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1755 sb_count * CCP_SB_BYTES);
1756 }
1757
1758 if (sha->final && sha->opad) {
1759 /* HMAC operation, recursively perform final SHA */
1760 struct ccp_cmd hmac_cmd;
1761 struct scatterlist sg;
1762 u8 *hmac_buf;
1763
1764 if (sha->opad_len != block_size) {
1765 ret = -EINVAL;
1766 goto e_data;
1767 }
1768
1769 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1770 if (!hmac_buf) {
1771 ret = -ENOMEM;
1772 goto e_data;
1773 }
1774 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1775
1776 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1777 switch (sha->type) {
1778 case CCP_SHA_TYPE_1:
1779 case CCP_SHA_TYPE_224:
1780 case CCP_SHA_TYPE_256:
1781 memcpy(hmac_buf + block_size,
1782 ctx.address + ooffset,
1783 digest_size);
1784 break;
1785 case CCP_SHA_TYPE_384:
1786 case CCP_SHA_TYPE_512:
1787 memcpy(hmac_buf + block_size,
1788 ctx.address + LSB_ITEM_SIZE + ooffset,
1789 LSB_ITEM_SIZE);
1790 memcpy(hmac_buf + block_size +
1791 (LSB_ITEM_SIZE - ooffset),
1792 ctx.address,
1793 LSB_ITEM_SIZE);
1794 break;
1795 default:
1796 kfree(hmac_buf);
1797 ret = -EINVAL;
1798 goto e_data;
1799 }
1800
1801 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1802 hmac_cmd.engine = CCP_ENGINE_SHA;
1803 hmac_cmd.u.sha.type = sha->type;
1804 hmac_cmd.u.sha.ctx = sha->ctx;
1805 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1806 hmac_cmd.u.sha.src = &sg;
1807 hmac_cmd.u.sha.src_len = block_size + digest_size;
1808 hmac_cmd.u.sha.opad = NULL;
1809 hmac_cmd.u.sha.opad_len = 0;
1810 hmac_cmd.u.sha.first = 1;
1811 hmac_cmd.u.sha.final = 1;
1812 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1813
1814 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1815 if (ret)
1816 cmd->engine_error = hmac_cmd.engine_error;
1817
1818 kfree(hmac_buf);
1819 }
1820
1821e_data:
1822 if (sha->src)
1823 ccp_free_data(&src, cmd_q);
1824
1825e_ctx:
1826 ccp_dm_free(&ctx);
1827
1828 return ret;
1829}
1830
1831static noinline_for_stack int
1832ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1833{
1834 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1835 struct ccp_dm_workarea exp, src, dst;
1836 struct ccp_op op;
1837 unsigned int sb_count, i_len, o_len;
1838 int ret;
1839
1840 /* Check against the maximum allowable size, in bits */
1841 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1842 return -EINVAL;
1843
1844 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1845 return -EINVAL;
1846
1847 memset(&op, 0, sizeof(op));
1848 op.cmd_q = cmd_q;
1849 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1850
1851 /* The RSA modulus must precede the message being acted upon, so
1852 * it must be copied to a DMA area where the message and the
1853 * modulus can be concatenated. Therefore the input buffer
1854 * length required is twice the output buffer length (which
1855 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1856 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1857 * required.
1858 */
1859 o_len = 32 * ((rsa->key_size + 255) / 256);
1860 i_len = o_len * 2;
1861
1862 sb_count = 0;
1863 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1864 /* sb_count is the number of storage block slots required
1865 * for the modulus.
1866 */
1867 sb_count = o_len / CCP_SB_BYTES;
1868 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1869 sb_count);
1870 if (!op.sb_key)
1871 return -EIO;
1872 } else {
1873 /* A version 5 device allows a modulus size that will not fit
1874 * in the LSB, so the command will transfer it from memory.
1875 * Set the sb key to the default, even though it's not used.
1876 */
1877 op.sb_key = cmd_q->sb_key;
1878 }
1879
1880 /* The RSA exponent must be in little endian format. Reverse its
1881 * byte order.
1882 */
1883 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1884 if (ret)
1885 goto e_sb;
1886
1887 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1888 if (ret)
1889 goto e_exp;
1890
1891 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1892 /* Copy the exponent to the local storage block, using
1893 * as many 32-byte blocks as were allocated above. It's
1894 * already little endian, so no further change is required.
1895 */
1896 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1897 CCP_PASSTHRU_BYTESWAP_NOOP);
1898 if (ret) {
1899 cmd->engine_error = cmd_q->cmd_error;
1900 goto e_exp;
1901 }
1902 } else {
1903 /* The exponent can be retrieved from memory via DMA. */
1904 op.exp.u.dma.address = exp.dma.address;
1905 op.exp.u.dma.offset = 0;
1906 }
1907
1908 /* Concatenate the modulus and the message. Both the modulus and
1909 * the operands must be in little endian format. Since the input
1910 * is in big endian format it must be converted.
1911 */
1912 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1913 if (ret)
1914 goto e_exp;
1915
1916 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1917 if (ret)
1918 goto e_src;
1919 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1920 if (ret)
1921 goto e_src;
1922
1923 /* Prepare the output area for the operation */
1924 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1925 if (ret)
1926 goto e_src;
1927
1928 op.soc = 1;
1929 op.src.u.dma.address = src.dma.address;
1930 op.src.u.dma.offset = 0;
1931 op.src.u.dma.length = i_len;
1932 op.dst.u.dma.address = dst.dma.address;
1933 op.dst.u.dma.offset = 0;
1934 op.dst.u.dma.length = o_len;
1935
1936 op.u.rsa.mod_size = rsa->key_size;
1937 op.u.rsa.input_len = i_len;
1938
1939 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1940 if (ret) {
1941 cmd->engine_error = cmd_q->cmd_error;
1942 goto e_dst;
1943 }
1944
1945 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1946
1947e_dst:
1948 ccp_dm_free(&dst);
1949
1950e_src:
1951 ccp_dm_free(&src);
1952
1953e_exp:
1954 ccp_dm_free(&exp);
1955
1956e_sb:
1957 if (sb_count)
1958 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1959
1960 return ret;
1961}
1962
1963static noinline_for_stack int
1964ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1965{
1966 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1967 struct ccp_dm_workarea mask;
1968 struct ccp_data src, dst;
1969 struct ccp_op op;
1970 bool in_place = false;
1971 unsigned int i;
1972 int ret = 0;
1973
1974 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1975 return -EINVAL;
1976
1977 if (!pt->src || !pt->dst)
1978 return -EINVAL;
1979
1980 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1982 return -EINVAL;
1983 if (!pt->mask)
1984 return -EINVAL;
1985 }
1986
1987 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1988
1989 memset(&op, 0, sizeof(op));
1990 op.cmd_q = cmd_q;
1991 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1992
1993 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1994 /* Load the mask */
1995 op.sb_key = cmd_q->sb_key;
1996
1997 ret = ccp_init_dm_workarea(&mask, cmd_q,
1998 CCP_PASSTHRU_SB_COUNT *
1999 CCP_SB_BYTES,
2000 DMA_TO_DEVICE);
2001 if (ret)
2002 return ret;
2003
2004 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2005 if (ret)
2006 goto e_mask;
2007 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2008 CCP_PASSTHRU_BYTESWAP_NOOP);
2009 if (ret) {
2010 cmd->engine_error = cmd_q->cmd_error;
2011 goto e_mask;
2012 }
2013 }
2014
2015 /* Prepare the input and output data workareas. For in-place
2016 * operations we need to set the dma direction to BIDIRECTIONAL
2017 * and copy the src workarea to the dst workarea.
2018 */
2019 if (sg_virt(pt->src) == sg_virt(pt->dst))
2020 in_place = true;
2021
2022 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2023 CCP_PASSTHRU_MASKSIZE,
2024 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2025 if (ret)
2026 goto e_mask;
2027
2028 if (in_place) {
2029 dst = src;
2030 } else {
2031 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2032 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2033 if (ret)
2034 goto e_src;
2035 }
2036
2037 /* Send data to the CCP Passthru engine
2038 * Because the CCP engine works on a single source and destination
2039 * dma address at a time, each entry in the source scatterlist
2040 * (after the dma_map_sg call) must be less than or equal to the
2041 * (remaining) length in the destination scatterlist entry and the
2042 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2043 */
2044 dst.sg_wa.sg_used = 0;
2045 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2046 if (!dst.sg_wa.sg ||
2047 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2048 ret = -EINVAL;
2049 goto e_dst;
2050 }
2051
2052 if (i == src.sg_wa.dma_count) {
2053 op.eom = 1;
2054 op.soc = 1;
2055 }
2056
2057 op.src.type = CCP_MEMTYPE_SYSTEM;
2058 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2059 op.src.u.dma.offset = 0;
2060 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2061
2062 op.dst.type = CCP_MEMTYPE_SYSTEM;
2063 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2064 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2065 op.dst.u.dma.length = op.src.u.dma.length;
2066
2067 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2068 if (ret) {
2069 cmd->engine_error = cmd_q->cmd_error;
2070 goto e_dst;
2071 }
2072
2073 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2074 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2075 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2076 dst.sg_wa.sg_used = 0;
2077 }
2078 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2079 }
2080
2081e_dst:
2082 if (!in_place)
2083 ccp_free_data(&dst, cmd_q);
2084
2085e_src:
2086 ccp_free_data(&src, cmd_q);
2087
2088e_mask:
2089 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2090 ccp_dm_free(&mask);
2091
2092 return ret;
2093}
2094
2095static noinline_for_stack int
2096ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2097 struct ccp_cmd *cmd)
2098{
2099 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2100 struct ccp_dm_workarea mask;
2101 struct ccp_op op;
2102 int ret;
2103
2104 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2105 return -EINVAL;
2106
2107 if (!pt->src_dma || !pt->dst_dma)
2108 return -EINVAL;
2109
2110 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2111 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2112 return -EINVAL;
2113 if (!pt->mask)
2114 return -EINVAL;
2115 }
2116
2117 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2118
2119 memset(&op, 0, sizeof(op));
2120 op.cmd_q = cmd_q;
2121 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2122
2123 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2124 /* Load the mask */
2125 op.sb_key = cmd_q->sb_key;
2126
2127 mask.length = pt->mask_len;
2128 mask.dma.address = pt->mask;
2129 mask.dma.length = pt->mask_len;
2130
2131 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2132 CCP_PASSTHRU_BYTESWAP_NOOP);
2133 if (ret) {
2134 cmd->engine_error = cmd_q->cmd_error;
2135 return ret;
2136 }
2137 }
2138
2139 /* Send data to the CCP Passthru engine */
2140 op.eom = 1;
2141 op.soc = 1;
2142
2143 op.src.type = CCP_MEMTYPE_SYSTEM;
2144 op.src.u.dma.address = pt->src_dma;
2145 op.src.u.dma.offset = 0;
2146 op.src.u.dma.length = pt->src_len;
2147
2148 op.dst.type = CCP_MEMTYPE_SYSTEM;
2149 op.dst.u.dma.address = pt->dst_dma;
2150 op.dst.u.dma.offset = 0;
2151 op.dst.u.dma.length = pt->src_len;
2152
2153 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2154 if (ret)
2155 cmd->engine_error = cmd_q->cmd_error;
2156
2157 return ret;
2158}
2159
2160static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2161{
2162 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2163 struct ccp_dm_workarea src, dst;
2164 struct ccp_op op;
2165 int ret;
2166 u8 *save;
2167
2168 if (!ecc->u.mm.operand_1 ||
2169 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2170 return -EINVAL;
2171
2172 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2173 if (!ecc->u.mm.operand_2 ||
2174 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2175 return -EINVAL;
2176
2177 if (!ecc->u.mm.result ||
2178 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2179 return -EINVAL;
2180
2181 memset(&op, 0, sizeof(op));
2182 op.cmd_q = cmd_q;
2183 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2184
2185 /* Concatenate the modulus and the operands. Both the modulus and
2186 * the operands must be in little endian format. Since the input
2187 * is in big endian format it must be converted and placed in a
2188 * fixed length buffer.
2189 */
2190 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2191 DMA_TO_DEVICE);
2192 if (ret)
2193 return ret;
2194
2195 /* Save the workarea address since it is updated in order to perform
2196 * the concatenation
2197 */
2198 save = src.address;
2199
2200 /* Copy the ECC modulus */
2201 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2202 if (ret)
2203 goto e_src;
2204 src.address += CCP_ECC_OPERAND_SIZE;
2205
2206 /* Copy the first operand */
2207 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2208 ecc->u.mm.operand_1_len);
2209 if (ret)
2210 goto e_src;
2211 src.address += CCP_ECC_OPERAND_SIZE;
2212
2213 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2214 /* Copy the second operand */
2215 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2216 ecc->u.mm.operand_2_len);
2217 if (ret)
2218 goto e_src;
2219 src.address += CCP_ECC_OPERAND_SIZE;
2220 }
2221
2222 /* Restore the workarea address */
2223 src.address = save;
2224
2225 /* Prepare the output area for the operation */
2226 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2227 DMA_FROM_DEVICE);
2228 if (ret)
2229 goto e_src;
2230
2231 op.soc = 1;
2232 op.src.u.dma.address = src.dma.address;
2233 op.src.u.dma.offset = 0;
2234 op.src.u.dma.length = src.length;
2235 op.dst.u.dma.address = dst.dma.address;
2236 op.dst.u.dma.offset = 0;
2237 op.dst.u.dma.length = dst.length;
2238
2239 op.u.ecc.function = cmd->u.ecc.function;
2240
2241 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2242 if (ret) {
2243 cmd->engine_error = cmd_q->cmd_error;
2244 goto e_dst;
2245 }
2246
2247 ecc->ecc_result = le16_to_cpup(
2248 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2249 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2250 ret = -EIO;
2251 goto e_dst;
2252 }
2253
2254 /* Save the ECC result */
2255 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2256 CCP_ECC_MODULUS_BYTES);
2257
2258e_dst:
2259 ccp_dm_free(&dst);
2260
2261e_src:
2262 ccp_dm_free(&src);
2263
2264 return ret;
2265}
2266
2267static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2268{
2269 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2270 struct ccp_dm_workarea src, dst;
2271 struct ccp_op op;
2272 int ret;
2273 u8 *save;
2274
2275 if (!ecc->u.pm.point_1.x ||
2276 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2277 !ecc->u.pm.point_1.y ||
2278 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2279 return -EINVAL;
2280
2281 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2282 if (!ecc->u.pm.point_2.x ||
2283 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2284 !ecc->u.pm.point_2.y ||
2285 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2286 return -EINVAL;
2287 } else {
2288 if (!ecc->u.pm.domain_a ||
2289 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2290 return -EINVAL;
2291
2292 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2293 if (!ecc->u.pm.scalar ||
2294 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2295 return -EINVAL;
2296 }
2297
2298 if (!ecc->u.pm.result.x ||
2299 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2300 !ecc->u.pm.result.y ||
2301 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2302 return -EINVAL;
2303
2304 memset(&op, 0, sizeof(op));
2305 op.cmd_q = cmd_q;
2306 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2307
2308 /* Concatenate the modulus and the operands. Both the modulus and
2309 * the operands must be in little endian format. Since the input
2310 * is in big endian format it must be converted and placed in a
2311 * fixed length buffer.
2312 */
2313 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2314 DMA_TO_DEVICE);
2315 if (ret)
2316 return ret;
2317
2318 /* Save the workarea address since it is updated in order to perform
2319 * the concatenation
2320 */
2321 save = src.address;
2322
2323 /* Copy the ECC modulus */
2324 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2325 if (ret)
2326 goto e_src;
2327 src.address += CCP_ECC_OPERAND_SIZE;
2328
2329 /* Copy the first point X and Y coordinate */
2330 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2331 ecc->u.pm.point_1.x_len);
2332 if (ret)
2333 goto e_src;
2334 src.address += CCP_ECC_OPERAND_SIZE;
2335 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2336 ecc->u.pm.point_1.y_len);
2337 if (ret)
2338 goto e_src;
2339 src.address += CCP_ECC_OPERAND_SIZE;
2340
2341 /* Set the first point Z coordinate to 1 */
2342 *src.address = 0x01;
2343 src.address += CCP_ECC_OPERAND_SIZE;
2344
2345 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2346 /* Copy the second point X and Y coordinate */
2347 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2348 ecc->u.pm.point_2.x_len);
2349 if (ret)
2350 goto e_src;
2351 src.address += CCP_ECC_OPERAND_SIZE;
2352 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2353 ecc->u.pm.point_2.y_len);
2354 if (ret)
2355 goto e_src;
2356 src.address += CCP_ECC_OPERAND_SIZE;
2357
2358 /* Set the second point Z coordinate to 1 */
2359 *src.address = 0x01;
2360 src.address += CCP_ECC_OPERAND_SIZE;
2361 } else {
2362 /* Copy the Domain "a" parameter */
2363 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2364 ecc->u.pm.domain_a_len);
2365 if (ret)
2366 goto e_src;
2367 src.address += CCP_ECC_OPERAND_SIZE;
2368
2369 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2370 /* Copy the scalar value */
2371 ret = ccp_reverse_set_dm_area(&src, 0,
2372 ecc->u.pm.scalar, 0,
2373 ecc->u.pm.scalar_len);
2374 if (ret)
2375 goto e_src;
2376 src.address += CCP_ECC_OPERAND_SIZE;
2377 }
2378 }
2379
2380 /* Restore the workarea address */
2381 src.address = save;
2382
2383 /* Prepare the output area for the operation */
2384 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2385 DMA_FROM_DEVICE);
2386 if (ret)
2387 goto e_src;
2388
2389 op.soc = 1;
2390 op.src.u.dma.address = src.dma.address;
2391 op.src.u.dma.offset = 0;
2392 op.src.u.dma.length = src.length;
2393 op.dst.u.dma.address = dst.dma.address;
2394 op.dst.u.dma.offset = 0;
2395 op.dst.u.dma.length = dst.length;
2396
2397 op.u.ecc.function = cmd->u.ecc.function;
2398
2399 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2400 if (ret) {
2401 cmd->engine_error = cmd_q->cmd_error;
2402 goto e_dst;
2403 }
2404
2405 ecc->ecc_result = le16_to_cpup(
2406 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2407 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2408 ret = -EIO;
2409 goto e_dst;
2410 }
2411
2412 /* Save the workarea address since it is updated as we walk through
2413 * to copy the point math result
2414 */
2415 save = dst.address;
2416
2417 /* Save the ECC result X and Y coordinates */
2418 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2419 CCP_ECC_MODULUS_BYTES);
2420 dst.address += CCP_ECC_OUTPUT_SIZE;
2421 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2422 CCP_ECC_MODULUS_BYTES);
2423
2424 /* Restore the workarea address */
2425 dst.address = save;
2426
2427e_dst:
2428 ccp_dm_free(&dst);
2429
2430e_src:
2431 ccp_dm_free(&src);
2432
2433 return ret;
2434}
2435
2436static noinline_for_stack int
2437ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2438{
2439 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2440
2441 ecc->ecc_result = 0;
2442
2443 if (!ecc->mod ||
2444 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2445 return -EINVAL;
2446
2447 switch (ecc->function) {
2448 case CCP_ECC_FUNCTION_MMUL_384BIT:
2449 case CCP_ECC_FUNCTION_MADD_384BIT:
2450 case CCP_ECC_FUNCTION_MINV_384BIT:
2451 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2452
2453 case CCP_ECC_FUNCTION_PADD_384BIT:
2454 case CCP_ECC_FUNCTION_PMUL_384BIT:
2455 case CCP_ECC_FUNCTION_PDBL_384BIT:
2456 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2457
2458 default:
2459 return -EINVAL;
2460 }
2461}
2462
2463int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2464{
2465 int ret;
2466
2467 cmd->engine_error = 0;
2468 cmd_q->cmd_error = 0;
2469 cmd_q->int_rcvd = 0;
2470 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2471
2472 switch (cmd->engine) {
2473 case CCP_ENGINE_AES:
2474 switch (cmd->u.aes.mode) {
2475 case CCP_AES_MODE_CMAC:
2476 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2477 break;
2478 case CCP_AES_MODE_GCM:
2479 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2480 break;
2481 default:
2482 ret = ccp_run_aes_cmd(cmd_q, cmd);
2483 break;
2484 }
2485 break;
2486 case CCP_ENGINE_XTS_AES_128:
2487 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2488 break;
2489 case CCP_ENGINE_DES3:
2490 ret = ccp_run_des3_cmd(cmd_q, cmd);
2491 break;
2492 case CCP_ENGINE_SHA:
2493 ret = ccp_run_sha_cmd(cmd_q, cmd);
2494 break;
2495 case CCP_ENGINE_RSA:
2496 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2497 break;
2498 case CCP_ENGINE_PASSTHRU:
2499 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2500 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2501 else
2502 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2503 break;
2504 case CCP_ENGINE_ECC:
2505 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2506 break;
2507 default:
2508 ret = -EINVAL;
2509 }
2510
2511 return ret;
2512}