Loading...
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/pci.h>
16#include <linux/interrupt.h>
17#include <crypto/scatterwalk.h>
18#include <linux/ccp.h>
19
20#include "ccp-dev.h"
21
22/* SHA initial context values */
23static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
24 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
25 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
26 cpu_to_be32(SHA1_H4), 0, 0, 0,
27};
28
29static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
30 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
31 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
32 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
33 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
34};
35
36static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
37 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
38 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
39 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
40 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
41};
42
43static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
44{
45 int start;
46
47 for (;;) {
48 mutex_lock(&ccp->ksb_mutex);
49
50 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
51 ccp->ksb_count,
52 ccp->ksb_start,
53 count, 0);
54 if (start <= ccp->ksb_count) {
55 bitmap_set(ccp->ksb, start, count);
56
57 mutex_unlock(&ccp->ksb_mutex);
58 break;
59 }
60
61 ccp->ksb_avail = 0;
62
63 mutex_unlock(&ccp->ksb_mutex);
64
65 /* Wait for KSB entries to become available */
66 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
67 return 0;
68 }
69
70 return KSB_START + start;
71}
72
73static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
74 unsigned int count)
75{
76 if (!start)
77 return;
78
79 mutex_lock(&ccp->ksb_mutex);
80
81 bitmap_clear(ccp->ksb, start - KSB_START, count);
82
83 ccp->ksb_avail = 1;
84
85 mutex_unlock(&ccp->ksb_mutex);
86
87 wake_up_interruptible_all(&ccp->ksb_queue);
88}
89
90static u32 ccp_gen_jobid(struct ccp_device *ccp)
91{
92 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
93}
94
95static void ccp_sg_free(struct ccp_sg_workarea *wa)
96{
97 if (wa->dma_count)
98 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
99
100 wa->dma_count = 0;
101}
102
103static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
104 struct scatterlist *sg, u64 len,
105 enum dma_data_direction dma_dir)
106{
107 memset(wa, 0, sizeof(*wa));
108
109 wa->sg = sg;
110 if (!sg)
111 return 0;
112
113 wa->nents = sg_nents_for_len(sg, len);
114 if (wa->nents < 0)
115 return wa->nents;
116
117 wa->bytes_left = len;
118 wa->sg_used = 0;
119
120 if (len == 0)
121 return 0;
122
123 if (dma_dir == DMA_NONE)
124 return 0;
125
126 wa->dma_sg = sg;
127 wa->dma_dev = dev;
128 wa->dma_dir = dma_dir;
129 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
130 if (!wa->dma_count)
131 return -ENOMEM;
132
133 return 0;
134}
135
136static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
137{
138 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
139
140 if (!wa->sg)
141 return;
142
143 wa->sg_used += nbytes;
144 wa->bytes_left -= nbytes;
145 if (wa->sg_used == wa->sg->length) {
146 wa->sg = sg_next(wa->sg);
147 wa->sg_used = 0;
148 }
149}
150
151static void ccp_dm_free(struct ccp_dm_workarea *wa)
152{
153 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
154 if (wa->address)
155 dma_pool_free(wa->dma_pool, wa->address,
156 wa->dma.address);
157 } else {
158 if (wa->dma.address)
159 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
160 wa->dma.dir);
161 kfree(wa->address);
162 }
163
164 wa->address = NULL;
165 wa->dma.address = 0;
166}
167
168static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
169 struct ccp_cmd_queue *cmd_q,
170 unsigned int len,
171 enum dma_data_direction dir)
172{
173 memset(wa, 0, sizeof(*wa));
174
175 if (!len)
176 return 0;
177
178 wa->dev = cmd_q->ccp->dev;
179 wa->length = len;
180
181 if (len <= CCP_DMAPOOL_MAX_SIZE) {
182 wa->dma_pool = cmd_q->dma_pool;
183
184 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
185 &wa->dma.address);
186 if (!wa->address)
187 return -ENOMEM;
188
189 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
190
191 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
192 } else {
193 wa->address = kzalloc(len, GFP_KERNEL);
194 if (!wa->address)
195 return -ENOMEM;
196
197 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
198 dir);
199 if (!wa->dma.address)
200 return -ENOMEM;
201
202 wa->dma.length = len;
203 }
204 wa->dma.dir = dir;
205
206 return 0;
207}
208
209static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
210 struct scatterlist *sg, unsigned int sg_offset,
211 unsigned int len)
212{
213 WARN_ON(!wa->address);
214
215 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
216 0);
217}
218
219static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
220 struct scatterlist *sg, unsigned int sg_offset,
221 unsigned int len)
222{
223 WARN_ON(!wa->address);
224
225 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
226 1);
227}
228
229static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
230 struct scatterlist *sg,
231 unsigned int len, unsigned int se_len,
232 bool sign_extend)
233{
234 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
235 u8 buffer[CCP_REVERSE_BUF_SIZE];
236
237 if (WARN_ON(se_len > sizeof(buffer)))
238 return -EINVAL;
239
240 sg_offset = len;
241 dm_offset = 0;
242 nbytes = len;
243 while (nbytes) {
244 ksb_len = min_t(unsigned int, nbytes, se_len);
245 sg_offset -= ksb_len;
246
247 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
248 for (i = 0; i < ksb_len; i++)
249 wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
250
251 dm_offset += ksb_len;
252 nbytes -= ksb_len;
253
254 if ((ksb_len != se_len) && sign_extend) {
255 /* Must sign-extend to nearest sign-extend length */
256 if (wa->address[dm_offset - 1] & 0x80)
257 memset(wa->address + dm_offset, 0xff,
258 se_len - ksb_len);
259 }
260 }
261
262 return 0;
263}
264
265static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
266 struct scatterlist *sg,
267 unsigned int len)
268{
269 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
270 u8 buffer[CCP_REVERSE_BUF_SIZE];
271
272 sg_offset = 0;
273 dm_offset = len;
274 nbytes = len;
275 while (nbytes) {
276 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
277 dm_offset -= ksb_len;
278
279 for (i = 0; i < ksb_len; i++)
280 buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
281 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
282
283 sg_offset += ksb_len;
284 nbytes -= ksb_len;
285 }
286}
287
288static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
289{
290 ccp_dm_free(&data->dm_wa);
291 ccp_sg_free(&data->sg_wa);
292}
293
294static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
295 struct scatterlist *sg, u64 sg_len,
296 unsigned int dm_len,
297 enum dma_data_direction dir)
298{
299 int ret;
300
301 memset(data, 0, sizeof(*data));
302
303 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
304 dir);
305 if (ret)
306 goto e_err;
307
308 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
309 if (ret)
310 goto e_err;
311
312 return 0;
313
314e_err:
315 ccp_free_data(data, cmd_q);
316
317 return ret;
318}
319
320static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
321{
322 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
323 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
324 unsigned int buf_count, nbytes;
325
326 /* Clear the buffer if setting it */
327 if (!from)
328 memset(dm_wa->address, 0, dm_wa->length);
329
330 if (!sg_wa->sg)
331 return 0;
332
333 /* Perform the copy operation
334 * nbytes will always be <= UINT_MAX because dm_wa->length is
335 * an unsigned int
336 */
337 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
338 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
339 nbytes, from);
340
341 /* Update the structures and generate the count */
342 buf_count = 0;
343 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
344 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
345 dm_wa->length - buf_count);
346 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
347
348 buf_count += nbytes;
349 ccp_update_sg_workarea(sg_wa, nbytes);
350 }
351
352 return buf_count;
353}
354
355static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
356{
357 return ccp_queue_buf(data, 0);
358}
359
360static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
361{
362 return ccp_queue_buf(data, 1);
363}
364
365static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
366 struct ccp_op *op, unsigned int block_size,
367 bool blocksize_op)
368{
369 unsigned int sg_src_len, sg_dst_len, op_len;
370
371 /* The CCP can only DMA from/to one address each per operation. This
372 * requires that we find the smallest DMA area between the source
373 * and destination. The resulting len values will always be <= UINT_MAX
374 * because the dma length is an unsigned int.
375 */
376 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
377 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
378
379 if (dst) {
380 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
381 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
382 op_len = min(sg_src_len, sg_dst_len);
383 } else {
384 op_len = sg_src_len;
385 }
386
387 /* The data operation length will be at least block_size in length
388 * or the smaller of available sg room remaining for the source or
389 * the destination
390 */
391 op_len = max(op_len, block_size);
392
393 /* Unless we have to buffer data, there's no reason to wait */
394 op->soc = 0;
395
396 if (sg_src_len < block_size) {
397 /* Not enough data in the sg element, so it
398 * needs to be buffered into a blocksize chunk
399 */
400 int cp_len = ccp_fill_queue_buf(src);
401
402 op->soc = 1;
403 op->src.u.dma.address = src->dm_wa.dma.address;
404 op->src.u.dma.offset = 0;
405 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
406 } else {
407 /* Enough data in the sg element, but we need to
408 * adjust for any previously copied data
409 */
410 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
411 op->src.u.dma.offset = src->sg_wa.sg_used;
412 op->src.u.dma.length = op_len & ~(block_size - 1);
413
414 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
415 }
416
417 if (dst) {
418 if (sg_dst_len < block_size) {
419 /* Not enough room in the sg element or we're on the
420 * last piece of data (when using padding), so the
421 * output needs to be buffered into a blocksize chunk
422 */
423 op->soc = 1;
424 op->dst.u.dma.address = dst->dm_wa.dma.address;
425 op->dst.u.dma.offset = 0;
426 op->dst.u.dma.length = op->src.u.dma.length;
427 } else {
428 /* Enough room in the sg element, but we need to
429 * adjust for any previously used area
430 */
431 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
432 op->dst.u.dma.offset = dst->sg_wa.sg_used;
433 op->dst.u.dma.length = op->src.u.dma.length;
434 }
435 }
436}
437
438static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
439 struct ccp_op *op)
440{
441 op->init = 0;
442
443 if (dst) {
444 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
445 ccp_empty_queue_buf(dst);
446 else
447 ccp_update_sg_workarea(&dst->sg_wa,
448 op->dst.u.dma.length);
449 }
450}
451
452static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
453 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
454 u32 byte_swap, bool from)
455{
456 struct ccp_op op;
457
458 memset(&op, 0, sizeof(op));
459
460 op.cmd_q = cmd_q;
461 op.jobid = jobid;
462 op.eom = 1;
463
464 if (from) {
465 op.soc = 1;
466 op.src.type = CCP_MEMTYPE_KSB;
467 op.src.u.ksb = ksb;
468 op.dst.type = CCP_MEMTYPE_SYSTEM;
469 op.dst.u.dma.address = wa->dma.address;
470 op.dst.u.dma.length = wa->length;
471 } else {
472 op.src.type = CCP_MEMTYPE_SYSTEM;
473 op.src.u.dma.address = wa->dma.address;
474 op.src.u.dma.length = wa->length;
475 op.dst.type = CCP_MEMTYPE_KSB;
476 op.dst.u.ksb = ksb;
477 }
478
479 op.u.passthru.byte_swap = byte_swap;
480
481 return cmd_q->ccp->vdata->perform->perform_passthru(&op);
482}
483
484static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
485 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
486 u32 byte_swap)
487{
488 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
489}
490
491static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
492 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
493 u32 byte_swap)
494{
495 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
496}
497
498static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
499 struct ccp_cmd *cmd)
500{
501 struct ccp_aes_engine *aes = &cmd->u.aes;
502 struct ccp_dm_workarea key, ctx;
503 struct ccp_data src;
504 struct ccp_op op;
505 unsigned int dm_offset;
506 int ret;
507
508 if (!((aes->key_len == AES_KEYSIZE_128) ||
509 (aes->key_len == AES_KEYSIZE_192) ||
510 (aes->key_len == AES_KEYSIZE_256)))
511 return -EINVAL;
512
513 if (aes->src_len & (AES_BLOCK_SIZE - 1))
514 return -EINVAL;
515
516 if (aes->iv_len != AES_BLOCK_SIZE)
517 return -EINVAL;
518
519 if (!aes->key || !aes->iv || !aes->src)
520 return -EINVAL;
521
522 if (aes->cmac_final) {
523 if (aes->cmac_key_len != AES_BLOCK_SIZE)
524 return -EINVAL;
525
526 if (!aes->cmac_key)
527 return -EINVAL;
528 }
529
530 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
531 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
532
533 ret = -EIO;
534 memset(&op, 0, sizeof(op));
535 op.cmd_q = cmd_q;
536 op.jobid = ccp_gen_jobid(cmd_q->ccp);
537 op.ksb_key = cmd_q->ksb_key;
538 op.ksb_ctx = cmd_q->ksb_ctx;
539 op.init = 1;
540 op.u.aes.type = aes->type;
541 op.u.aes.mode = aes->mode;
542 op.u.aes.action = aes->action;
543
544 /* All supported key sizes fit in a single (32-byte) KSB entry
545 * and must be in little endian format. Use the 256-bit byte
546 * swap passthru option to convert from big endian to little
547 * endian.
548 */
549 ret = ccp_init_dm_workarea(&key, cmd_q,
550 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
551 DMA_TO_DEVICE);
552 if (ret)
553 return ret;
554
555 dm_offset = CCP_KSB_BYTES - aes->key_len;
556 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
557 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
558 CCP_PASSTHRU_BYTESWAP_256BIT);
559 if (ret) {
560 cmd->engine_error = cmd_q->cmd_error;
561 goto e_key;
562 }
563
564 /* The AES context fits in a single (32-byte) KSB entry and
565 * must be in little endian format. Use the 256-bit byte swap
566 * passthru option to convert from big endian to little endian.
567 */
568 ret = ccp_init_dm_workarea(&ctx, cmd_q,
569 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
570 DMA_BIDIRECTIONAL);
571 if (ret)
572 goto e_key;
573
574 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
575 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
576 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
577 CCP_PASSTHRU_BYTESWAP_256BIT);
578 if (ret) {
579 cmd->engine_error = cmd_q->cmd_error;
580 goto e_ctx;
581 }
582
583 /* Send data to the CCP AES engine */
584 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
585 AES_BLOCK_SIZE, DMA_TO_DEVICE);
586 if (ret)
587 goto e_ctx;
588
589 while (src.sg_wa.bytes_left) {
590 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
591 if (aes->cmac_final && !src.sg_wa.bytes_left) {
592 op.eom = 1;
593
594 /* Push the K1/K2 key to the CCP now */
595 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
596 op.ksb_ctx,
597 CCP_PASSTHRU_BYTESWAP_256BIT);
598 if (ret) {
599 cmd->engine_error = cmd_q->cmd_error;
600 goto e_src;
601 }
602
603 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
604 aes->cmac_key_len);
605 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
606 CCP_PASSTHRU_BYTESWAP_256BIT);
607 if (ret) {
608 cmd->engine_error = cmd_q->cmd_error;
609 goto e_src;
610 }
611 }
612
613 ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
614 if (ret) {
615 cmd->engine_error = cmd_q->cmd_error;
616 goto e_src;
617 }
618
619 ccp_process_data(&src, NULL, &op);
620 }
621
622 /* Retrieve the AES context - convert from LE to BE using
623 * 32-byte (256-bit) byteswapping
624 */
625 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
626 CCP_PASSTHRU_BYTESWAP_256BIT);
627 if (ret) {
628 cmd->engine_error = cmd_q->cmd_error;
629 goto e_src;
630 }
631
632 /* ...but we only need AES_BLOCK_SIZE bytes */
633 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
634 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
635
636e_src:
637 ccp_free_data(&src, cmd_q);
638
639e_ctx:
640 ccp_dm_free(&ctx);
641
642e_key:
643 ccp_dm_free(&key);
644
645 return ret;
646}
647
648static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
649{
650 struct ccp_aes_engine *aes = &cmd->u.aes;
651 struct ccp_dm_workarea key, ctx;
652 struct ccp_data src, dst;
653 struct ccp_op op;
654 unsigned int dm_offset;
655 bool in_place = false;
656 int ret;
657
658 if (aes->mode == CCP_AES_MODE_CMAC)
659 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
660
661 if (!((aes->key_len == AES_KEYSIZE_128) ||
662 (aes->key_len == AES_KEYSIZE_192) ||
663 (aes->key_len == AES_KEYSIZE_256)))
664 return -EINVAL;
665
666 if (((aes->mode == CCP_AES_MODE_ECB) ||
667 (aes->mode == CCP_AES_MODE_CBC) ||
668 (aes->mode == CCP_AES_MODE_CFB)) &&
669 (aes->src_len & (AES_BLOCK_SIZE - 1)))
670 return -EINVAL;
671
672 if (!aes->key || !aes->src || !aes->dst)
673 return -EINVAL;
674
675 if (aes->mode != CCP_AES_MODE_ECB) {
676 if (aes->iv_len != AES_BLOCK_SIZE)
677 return -EINVAL;
678
679 if (!aes->iv)
680 return -EINVAL;
681 }
682
683 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
684 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
685
686 ret = -EIO;
687 memset(&op, 0, sizeof(op));
688 op.cmd_q = cmd_q;
689 op.jobid = ccp_gen_jobid(cmd_q->ccp);
690 op.ksb_key = cmd_q->ksb_key;
691 op.ksb_ctx = cmd_q->ksb_ctx;
692 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
693 op.u.aes.type = aes->type;
694 op.u.aes.mode = aes->mode;
695 op.u.aes.action = aes->action;
696
697 /* All supported key sizes fit in a single (32-byte) KSB entry
698 * and must be in little endian format. Use the 256-bit byte
699 * swap passthru option to convert from big endian to little
700 * endian.
701 */
702 ret = ccp_init_dm_workarea(&key, cmd_q,
703 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
704 DMA_TO_DEVICE);
705 if (ret)
706 return ret;
707
708 dm_offset = CCP_KSB_BYTES - aes->key_len;
709 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
710 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
711 CCP_PASSTHRU_BYTESWAP_256BIT);
712 if (ret) {
713 cmd->engine_error = cmd_q->cmd_error;
714 goto e_key;
715 }
716
717 /* The AES context fits in a single (32-byte) KSB entry and
718 * must be in little endian format. Use the 256-bit byte swap
719 * passthru option to convert from big endian to little endian.
720 */
721 ret = ccp_init_dm_workarea(&ctx, cmd_q,
722 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
723 DMA_BIDIRECTIONAL);
724 if (ret)
725 goto e_key;
726
727 if (aes->mode != CCP_AES_MODE_ECB) {
728 /* Load the AES context - conver to LE */
729 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
730 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
731 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
732 CCP_PASSTHRU_BYTESWAP_256BIT);
733 if (ret) {
734 cmd->engine_error = cmd_q->cmd_error;
735 goto e_ctx;
736 }
737 }
738
739 /* Prepare the input and output data workareas. For in-place
740 * operations we need to set the dma direction to BIDIRECTIONAL
741 * and copy the src workarea to the dst workarea.
742 */
743 if (sg_virt(aes->src) == sg_virt(aes->dst))
744 in_place = true;
745
746 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
747 AES_BLOCK_SIZE,
748 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
749 if (ret)
750 goto e_ctx;
751
752 if (in_place) {
753 dst = src;
754 } else {
755 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
756 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
757 if (ret)
758 goto e_src;
759 }
760
761 /* Send data to the CCP AES engine */
762 while (src.sg_wa.bytes_left) {
763 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
764 if (!src.sg_wa.bytes_left) {
765 op.eom = 1;
766
767 /* Since we don't retrieve the AES context in ECB
768 * mode we have to wait for the operation to complete
769 * on the last piece of data
770 */
771 if (aes->mode == CCP_AES_MODE_ECB)
772 op.soc = 1;
773 }
774
775 ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
776 if (ret) {
777 cmd->engine_error = cmd_q->cmd_error;
778 goto e_dst;
779 }
780
781 ccp_process_data(&src, &dst, &op);
782 }
783
784 if (aes->mode != CCP_AES_MODE_ECB) {
785 /* Retrieve the AES context - convert from LE to BE using
786 * 32-byte (256-bit) byteswapping
787 */
788 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
789 CCP_PASSTHRU_BYTESWAP_256BIT);
790 if (ret) {
791 cmd->engine_error = cmd_q->cmd_error;
792 goto e_dst;
793 }
794
795 /* ...but we only need AES_BLOCK_SIZE bytes */
796 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
797 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
798 }
799
800e_dst:
801 if (!in_place)
802 ccp_free_data(&dst, cmd_q);
803
804e_src:
805 ccp_free_data(&src, cmd_q);
806
807e_ctx:
808 ccp_dm_free(&ctx);
809
810e_key:
811 ccp_dm_free(&key);
812
813 return ret;
814}
815
816static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
817 struct ccp_cmd *cmd)
818{
819 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
820 struct ccp_dm_workarea key, ctx;
821 struct ccp_data src, dst;
822 struct ccp_op op;
823 unsigned int unit_size, dm_offset;
824 bool in_place = false;
825 int ret;
826
827 switch (xts->unit_size) {
828 case CCP_XTS_AES_UNIT_SIZE_16:
829 unit_size = 16;
830 break;
831 case CCP_XTS_AES_UNIT_SIZE_512:
832 unit_size = 512;
833 break;
834 case CCP_XTS_AES_UNIT_SIZE_1024:
835 unit_size = 1024;
836 break;
837 case CCP_XTS_AES_UNIT_SIZE_2048:
838 unit_size = 2048;
839 break;
840 case CCP_XTS_AES_UNIT_SIZE_4096:
841 unit_size = 4096;
842 break;
843
844 default:
845 return -EINVAL;
846 }
847
848 if (xts->key_len != AES_KEYSIZE_128)
849 return -EINVAL;
850
851 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
852 return -EINVAL;
853
854 if (xts->iv_len != AES_BLOCK_SIZE)
855 return -EINVAL;
856
857 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
858 return -EINVAL;
859
860 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
861 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
862
863 ret = -EIO;
864 memset(&op, 0, sizeof(op));
865 op.cmd_q = cmd_q;
866 op.jobid = ccp_gen_jobid(cmd_q->ccp);
867 op.ksb_key = cmd_q->ksb_key;
868 op.ksb_ctx = cmd_q->ksb_ctx;
869 op.init = 1;
870 op.u.xts.action = xts->action;
871 op.u.xts.unit_size = xts->unit_size;
872
873 /* All supported key sizes fit in a single (32-byte) KSB entry
874 * and must be in little endian format. Use the 256-bit byte
875 * swap passthru option to convert from big endian to little
876 * endian.
877 */
878 ret = ccp_init_dm_workarea(&key, cmd_q,
879 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
880 DMA_TO_DEVICE);
881 if (ret)
882 return ret;
883
884 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
885 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
886 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
887 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
888 CCP_PASSTHRU_BYTESWAP_256BIT);
889 if (ret) {
890 cmd->engine_error = cmd_q->cmd_error;
891 goto e_key;
892 }
893
894 /* The AES context fits in a single (32-byte) KSB entry and
895 * for XTS is already in little endian format so no byte swapping
896 * is needed.
897 */
898 ret = ccp_init_dm_workarea(&ctx, cmd_q,
899 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
900 DMA_BIDIRECTIONAL);
901 if (ret)
902 goto e_key;
903
904 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
905 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
906 CCP_PASSTHRU_BYTESWAP_NOOP);
907 if (ret) {
908 cmd->engine_error = cmd_q->cmd_error;
909 goto e_ctx;
910 }
911
912 /* Prepare the input and output data workareas. For in-place
913 * operations we need to set the dma direction to BIDIRECTIONAL
914 * and copy the src workarea to the dst workarea.
915 */
916 if (sg_virt(xts->src) == sg_virt(xts->dst))
917 in_place = true;
918
919 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
920 unit_size,
921 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
922 if (ret)
923 goto e_ctx;
924
925 if (in_place) {
926 dst = src;
927 } else {
928 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
929 unit_size, DMA_FROM_DEVICE);
930 if (ret)
931 goto e_src;
932 }
933
934 /* Send data to the CCP AES engine */
935 while (src.sg_wa.bytes_left) {
936 ccp_prepare_data(&src, &dst, &op, unit_size, true);
937 if (!src.sg_wa.bytes_left)
938 op.eom = 1;
939
940 ret = cmd_q->ccp->vdata->perform->perform_xts_aes(&op);
941 if (ret) {
942 cmd->engine_error = cmd_q->cmd_error;
943 goto e_dst;
944 }
945
946 ccp_process_data(&src, &dst, &op);
947 }
948
949 /* Retrieve the AES context - convert from LE to BE using
950 * 32-byte (256-bit) byteswapping
951 */
952 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
953 CCP_PASSTHRU_BYTESWAP_256BIT);
954 if (ret) {
955 cmd->engine_error = cmd_q->cmd_error;
956 goto e_dst;
957 }
958
959 /* ...but we only need AES_BLOCK_SIZE bytes */
960 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
961 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
962
963e_dst:
964 if (!in_place)
965 ccp_free_data(&dst, cmd_q);
966
967e_src:
968 ccp_free_data(&src, cmd_q);
969
970e_ctx:
971 ccp_dm_free(&ctx);
972
973e_key:
974 ccp_dm_free(&key);
975
976 return ret;
977}
978
979static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
980{
981 struct ccp_sha_engine *sha = &cmd->u.sha;
982 struct ccp_dm_workarea ctx;
983 struct ccp_data src;
984 struct ccp_op op;
985 int ret;
986
987 if (sha->ctx_len != CCP_SHA_CTXSIZE)
988 return -EINVAL;
989
990 if (!sha->ctx)
991 return -EINVAL;
992
993 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
994 return -EINVAL;
995
996 if (!sha->src_len) {
997 const u8 *sha_zero;
998
999 /* Not final, just return */
1000 if (!sha->final)
1001 return 0;
1002
1003 /* CCP can't do a zero length sha operation so the caller
1004 * must buffer the data.
1005 */
1006 if (sha->msg_bits)
1007 return -EINVAL;
1008
1009 /* The CCP cannot perform zero-length sha operations so the
1010 * caller is required to buffer data for the final operation.
1011 * However, a sha operation for a message with a total length
1012 * of zero is valid so known values are required to supply
1013 * the result.
1014 */
1015 switch (sha->type) {
1016 case CCP_SHA_TYPE_1:
1017 sha_zero = sha1_zero_message_hash;
1018 break;
1019 case CCP_SHA_TYPE_224:
1020 sha_zero = sha224_zero_message_hash;
1021 break;
1022 case CCP_SHA_TYPE_256:
1023 sha_zero = sha256_zero_message_hash;
1024 break;
1025 default:
1026 return -EINVAL;
1027 }
1028
1029 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1030 sha->ctx_len, 1);
1031
1032 return 0;
1033 }
1034
1035 if (!sha->src)
1036 return -EINVAL;
1037
1038 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1039
1040 memset(&op, 0, sizeof(op));
1041 op.cmd_q = cmd_q;
1042 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1043 op.ksb_ctx = cmd_q->ksb_ctx;
1044 op.u.sha.type = sha->type;
1045 op.u.sha.msg_bits = sha->msg_bits;
1046
1047 /* The SHA context fits in a single (32-byte) KSB entry and
1048 * must be in little endian format. Use the 256-bit byte swap
1049 * passthru option to convert from big endian to little endian.
1050 */
1051 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1052 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1053 DMA_BIDIRECTIONAL);
1054 if (ret)
1055 return ret;
1056
1057 if (sha->first) {
1058 const __be32 *init;
1059
1060 switch (sha->type) {
1061 case CCP_SHA_TYPE_1:
1062 init = ccp_sha1_init;
1063 break;
1064 case CCP_SHA_TYPE_224:
1065 init = ccp_sha224_init;
1066 break;
1067 case CCP_SHA_TYPE_256:
1068 init = ccp_sha256_init;
1069 break;
1070 default:
1071 ret = -EINVAL;
1072 goto e_ctx;
1073 }
1074 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1075 } else {
1076 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1077 }
1078
1079 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1080 CCP_PASSTHRU_BYTESWAP_256BIT);
1081 if (ret) {
1082 cmd->engine_error = cmd_q->cmd_error;
1083 goto e_ctx;
1084 }
1085
1086 /* Send data to the CCP SHA engine */
1087 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1088 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1089 if (ret)
1090 goto e_ctx;
1091
1092 while (src.sg_wa.bytes_left) {
1093 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1094 if (sha->final && !src.sg_wa.bytes_left)
1095 op.eom = 1;
1096
1097 ret = cmd_q->ccp->vdata->perform->perform_sha(&op);
1098 if (ret) {
1099 cmd->engine_error = cmd_q->cmd_error;
1100 goto e_data;
1101 }
1102
1103 ccp_process_data(&src, NULL, &op);
1104 }
1105
1106 /* Retrieve the SHA context - convert from LE to BE using
1107 * 32-byte (256-bit) byteswapping to BE
1108 */
1109 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1110 CCP_PASSTHRU_BYTESWAP_256BIT);
1111 if (ret) {
1112 cmd->engine_error = cmd_q->cmd_error;
1113 goto e_data;
1114 }
1115
1116 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1117
1118 if (sha->final && sha->opad) {
1119 /* HMAC operation, recursively perform final SHA */
1120 struct ccp_cmd hmac_cmd;
1121 struct scatterlist sg;
1122 u64 block_size, digest_size;
1123 u8 *hmac_buf;
1124
1125 switch (sha->type) {
1126 case CCP_SHA_TYPE_1:
1127 block_size = SHA1_BLOCK_SIZE;
1128 digest_size = SHA1_DIGEST_SIZE;
1129 break;
1130 case CCP_SHA_TYPE_224:
1131 block_size = SHA224_BLOCK_SIZE;
1132 digest_size = SHA224_DIGEST_SIZE;
1133 break;
1134 case CCP_SHA_TYPE_256:
1135 block_size = SHA256_BLOCK_SIZE;
1136 digest_size = SHA256_DIGEST_SIZE;
1137 break;
1138 default:
1139 ret = -EINVAL;
1140 goto e_data;
1141 }
1142
1143 if (sha->opad_len != block_size) {
1144 ret = -EINVAL;
1145 goto e_data;
1146 }
1147
1148 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1149 if (!hmac_buf) {
1150 ret = -ENOMEM;
1151 goto e_data;
1152 }
1153 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1154
1155 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1156 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1157
1158 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1159 hmac_cmd.engine = CCP_ENGINE_SHA;
1160 hmac_cmd.u.sha.type = sha->type;
1161 hmac_cmd.u.sha.ctx = sha->ctx;
1162 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1163 hmac_cmd.u.sha.src = &sg;
1164 hmac_cmd.u.sha.src_len = block_size + digest_size;
1165 hmac_cmd.u.sha.opad = NULL;
1166 hmac_cmd.u.sha.opad_len = 0;
1167 hmac_cmd.u.sha.first = 1;
1168 hmac_cmd.u.sha.final = 1;
1169 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1170
1171 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1172 if (ret)
1173 cmd->engine_error = hmac_cmd.engine_error;
1174
1175 kfree(hmac_buf);
1176 }
1177
1178e_data:
1179 ccp_free_data(&src, cmd_q);
1180
1181e_ctx:
1182 ccp_dm_free(&ctx);
1183
1184 return ret;
1185}
1186
1187static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1188{
1189 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1190 struct ccp_dm_workarea exp, src;
1191 struct ccp_data dst;
1192 struct ccp_op op;
1193 unsigned int ksb_count, i_len, o_len;
1194 int ret;
1195
1196 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1197 return -EINVAL;
1198
1199 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1200 return -EINVAL;
1201
1202 /* The RSA modulus must precede the message being acted upon, so
1203 * it must be copied to a DMA area where the message and the
1204 * modulus can be concatenated. Therefore the input buffer
1205 * length required is twice the output buffer length (which
1206 * must be a multiple of 256-bits).
1207 */
1208 o_len = ((rsa->key_size + 255) / 256) * 32;
1209 i_len = o_len * 2;
1210
1211 ksb_count = o_len / CCP_KSB_BYTES;
1212
1213 memset(&op, 0, sizeof(op));
1214 op.cmd_q = cmd_q;
1215 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1216 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1217 if (!op.ksb_key)
1218 return -EIO;
1219
1220 /* The RSA exponent may span multiple (32-byte) KSB entries and must
1221 * be in little endian format. Reverse copy each 32-byte chunk
1222 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1223 * and each byte within that chunk and do not perform any byte swap
1224 * operations on the passthru operation.
1225 */
1226 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1227 if (ret)
1228 goto e_ksb;
1229
1230 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1231 CCP_KSB_BYTES, false);
1232 if (ret)
1233 goto e_exp;
1234 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1235 CCP_PASSTHRU_BYTESWAP_NOOP);
1236 if (ret) {
1237 cmd->engine_error = cmd_q->cmd_error;
1238 goto e_exp;
1239 }
1240
1241 /* Concatenate the modulus and the message. Both the modulus and
1242 * the operands must be in little endian format. Since the input
1243 * is in big endian format it must be converted.
1244 */
1245 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1246 if (ret)
1247 goto e_exp;
1248
1249 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1250 CCP_KSB_BYTES, false);
1251 if (ret)
1252 goto e_src;
1253 src.address += o_len; /* Adjust the address for the copy operation */
1254 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1255 CCP_KSB_BYTES, false);
1256 if (ret)
1257 goto e_src;
1258 src.address -= o_len; /* Reset the address to original value */
1259
1260 /* Prepare the output area for the operation */
1261 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1262 o_len, DMA_FROM_DEVICE);
1263 if (ret)
1264 goto e_src;
1265
1266 op.soc = 1;
1267 op.src.u.dma.address = src.dma.address;
1268 op.src.u.dma.offset = 0;
1269 op.src.u.dma.length = i_len;
1270 op.dst.u.dma.address = dst.dm_wa.dma.address;
1271 op.dst.u.dma.offset = 0;
1272 op.dst.u.dma.length = o_len;
1273
1274 op.u.rsa.mod_size = rsa->key_size;
1275 op.u.rsa.input_len = i_len;
1276
1277 ret = cmd_q->ccp->vdata->perform->perform_rsa(&op);
1278 if (ret) {
1279 cmd->engine_error = cmd_q->cmd_error;
1280 goto e_dst;
1281 }
1282
1283 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1284
1285e_dst:
1286 ccp_free_data(&dst, cmd_q);
1287
1288e_src:
1289 ccp_dm_free(&src);
1290
1291e_exp:
1292 ccp_dm_free(&exp);
1293
1294e_ksb:
1295 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1296
1297 return ret;
1298}
1299
1300static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1301 struct ccp_cmd *cmd)
1302{
1303 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1304 struct ccp_dm_workarea mask;
1305 struct ccp_data src, dst;
1306 struct ccp_op op;
1307 bool in_place = false;
1308 unsigned int i;
1309 int ret;
1310
1311 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1312 return -EINVAL;
1313
1314 if (!pt->src || !pt->dst)
1315 return -EINVAL;
1316
1317 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1318 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1319 return -EINVAL;
1320 if (!pt->mask)
1321 return -EINVAL;
1322 }
1323
1324 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1325
1326 memset(&op, 0, sizeof(op));
1327 op.cmd_q = cmd_q;
1328 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1329
1330 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1331 /* Load the mask */
1332 op.ksb_key = cmd_q->ksb_key;
1333
1334 ret = ccp_init_dm_workarea(&mask, cmd_q,
1335 CCP_PASSTHRU_KSB_COUNT *
1336 CCP_KSB_BYTES,
1337 DMA_TO_DEVICE);
1338 if (ret)
1339 return ret;
1340
1341 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1342 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1343 CCP_PASSTHRU_BYTESWAP_NOOP);
1344 if (ret) {
1345 cmd->engine_error = cmd_q->cmd_error;
1346 goto e_mask;
1347 }
1348 }
1349
1350 /* Prepare the input and output data workareas. For in-place
1351 * operations we need to set the dma direction to BIDIRECTIONAL
1352 * and copy the src workarea to the dst workarea.
1353 */
1354 if (sg_virt(pt->src) == sg_virt(pt->dst))
1355 in_place = true;
1356
1357 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1358 CCP_PASSTHRU_MASKSIZE,
1359 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1360 if (ret)
1361 goto e_mask;
1362
1363 if (in_place) {
1364 dst = src;
1365 } else {
1366 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1367 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1368 if (ret)
1369 goto e_src;
1370 }
1371
1372 /* Send data to the CCP Passthru engine
1373 * Because the CCP engine works on a single source and destination
1374 * dma address at a time, each entry in the source scatterlist
1375 * (after the dma_map_sg call) must be less than or equal to the
1376 * (remaining) length in the destination scatterlist entry and the
1377 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1378 */
1379 dst.sg_wa.sg_used = 0;
1380 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1381 if (!dst.sg_wa.sg ||
1382 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1383 ret = -EINVAL;
1384 goto e_dst;
1385 }
1386
1387 if (i == src.sg_wa.dma_count) {
1388 op.eom = 1;
1389 op.soc = 1;
1390 }
1391
1392 op.src.type = CCP_MEMTYPE_SYSTEM;
1393 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1394 op.src.u.dma.offset = 0;
1395 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1396
1397 op.dst.type = CCP_MEMTYPE_SYSTEM;
1398 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1399 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1400 op.dst.u.dma.length = op.src.u.dma.length;
1401
1402 ret = cmd_q->ccp->vdata->perform->perform_passthru(&op);
1403 if (ret) {
1404 cmd->engine_error = cmd_q->cmd_error;
1405 goto e_dst;
1406 }
1407
1408 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1409 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1410 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1411 dst.sg_wa.sg_used = 0;
1412 }
1413 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1414 }
1415
1416e_dst:
1417 if (!in_place)
1418 ccp_free_data(&dst, cmd_q);
1419
1420e_src:
1421 ccp_free_data(&src, cmd_q);
1422
1423e_mask:
1424 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1425 ccp_dm_free(&mask);
1426
1427 return ret;
1428}
1429
1430static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1431{
1432 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1433 struct ccp_dm_workarea src, dst;
1434 struct ccp_op op;
1435 int ret;
1436 u8 *save;
1437
1438 if (!ecc->u.mm.operand_1 ||
1439 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1440 return -EINVAL;
1441
1442 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1443 if (!ecc->u.mm.operand_2 ||
1444 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1445 return -EINVAL;
1446
1447 if (!ecc->u.mm.result ||
1448 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1449 return -EINVAL;
1450
1451 memset(&op, 0, sizeof(op));
1452 op.cmd_q = cmd_q;
1453 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1454
1455 /* Concatenate the modulus and the operands. Both the modulus and
1456 * the operands must be in little endian format. Since the input
1457 * is in big endian format it must be converted and placed in a
1458 * fixed length buffer.
1459 */
1460 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1461 DMA_TO_DEVICE);
1462 if (ret)
1463 return ret;
1464
1465 /* Save the workarea address since it is updated in order to perform
1466 * the concatenation
1467 */
1468 save = src.address;
1469
1470 /* Copy the ECC modulus */
1471 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1472 CCP_ECC_OPERAND_SIZE, false);
1473 if (ret)
1474 goto e_src;
1475 src.address += CCP_ECC_OPERAND_SIZE;
1476
1477 /* Copy the first operand */
1478 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1479 ecc->u.mm.operand_1_len,
1480 CCP_ECC_OPERAND_SIZE, false);
1481 if (ret)
1482 goto e_src;
1483 src.address += CCP_ECC_OPERAND_SIZE;
1484
1485 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1486 /* Copy the second operand */
1487 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1488 ecc->u.mm.operand_2_len,
1489 CCP_ECC_OPERAND_SIZE, false);
1490 if (ret)
1491 goto e_src;
1492 src.address += CCP_ECC_OPERAND_SIZE;
1493 }
1494
1495 /* Restore the workarea address */
1496 src.address = save;
1497
1498 /* Prepare the output area for the operation */
1499 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1500 DMA_FROM_DEVICE);
1501 if (ret)
1502 goto e_src;
1503
1504 op.soc = 1;
1505 op.src.u.dma.address = src.dma.address;
1506 op.src.u.dma.offset = 0;
1507 op.src.u.dma.length = src.length;
1508 op.dst.u.dma.address = dst.dma.address;
1509 op.dst.u.dma.offset = 0;
1510 op.dst.u.dma.length = dst.length;
1511
1512 op.u.ecc.function = cmd->u.ecc.function;
1513
1514 ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
1515 if (ret) {
1516 cmd->engine_error = cmd_q->cmd_error;
1517 goto e_dst;
1518 }
1519
1520 ecc->ecc_result = le16_to_cpup(
1521 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1522 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1523 ret = -EIO;
1524 goto e_dst;
1525 }
1526
1527 /* Save the ECC result */
1528 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1529
1530e_dst:
1531 ccp_dm_free(&dst);
1532
1533e_src:
1534 ccp_dm_free(&src);
1535
1536 return ret;
1537}
1538
1539static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1540{
1541 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1542 struct ccp_dm_workarea src, dst;
1543 struct ccp_op op;
1544 int ret;
1545 u8 *save;
1546
1547 if (!ecc->u.pm.point_1.x ||
1548 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1549 !ecc->u.pm.point_1.y ||
1550 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1551 return -EINVAL;
1552
1553 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1554 if (!ecc->u.pm.point_2.x ||
1555 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1556 !ecc->u.pm.point_2.y ||
1557 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1558 return -EINVAL;
1559 } else {
1560 if (!ecc->u.pm.domain_a ||
1561 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1562 return -EINVAL;
1563
1564 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1565 if (!ecc->u.pm.scalar ||
1566 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1567 return -EINVAL;
1568 }
1569
1570 if (!ecc->u.pm.result.x ||
1571 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1572 !ecc->u.pm.result.y ||
1573 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1574 return -EINVAL;
1575
1576 memset(&op, 0, sizeof(op));
1577 op.cmd_q = cmd_q;
1578 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1579
1580 /* Concatenate the modulus and the operands. Both the modulus and
1581 * the operands must be in little endian format. Since the input
1582 * is in big endian format it must be converted and placed in a
1583 * fixed length buffer.
1584 */
1585 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1586 DMA_TO_DEVICE);
1587 if (ret)
1588 return ret;
1589
1590 /* Save the workarea address since it is updated in order to perform
1591 * the concatenation
1592 */
1593 save = src.address;
1594
1595 /* Copy the ECC modulus */
1596 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1597 CCP_ECC_OPERAND_SIZE, false);
1598 if (ret)
1599 goto e_src;
1600 src.address += CCP_ECC_OPERAND_SIZE;
1601
1602 /* Copy the first point X and Y coordinate */
1603 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1604 ecc->u.pm.point_1.x_len,
1605 CCP_ECC_OPERAND_SIZE, false);
1606 if (ret)
1607 goto e_src;
1608 src.address += CCP_ECC_OPERAND_SIZE;
1609 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1610 ecc->u.pm.point_1.y_len,
1611 CCP_ECC_OPERAND_SIZE, false);
1612 if (ret)
1613 goto e_src;
1614 src.address += CCP_ECC_OPERAND_SIZE;
1615
1616 /* Set the first point Z coordianate to 1 */
1617 *src.address = 0x01;
1618 src.address += CCP_ECC_OPERAND_SIZE;
1619
1620 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1621 /* Copy the second point X and Y coordinate */
1622 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1623 ecc->u.pm.point_2.x_len,
1624 CCP_ECC_OPERAND_SIZE, false);
1625 if (ret)
1626 goto e_src;
1627 src.address += CCP_ECC_OPERAND_SIZE;
1628 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1629 ecc->u.pm.point_2.y_len,
1630 CCP_ECC_OPERAND_SIZE, false);
1631 if (ret)
1632 goto e_src;
1633 src.address += CCP_ECC_OPERAND_SIZE;
1634
1635 /* Set the second point Z coordianate to 1 */
1636 *src.address = 0x01;
1637 src.address += CCP_ECC_OPERAND_SIZE;
1638 } else {
1639 /* Copy the Domain "a" parameter */
1640 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1641 ecc->u.pm.domain_a_len,
1642 CCP_ECC_OPERAND_SIZE, false);
1643 if (ret)
1644 goto e_src;
1645 src.address += CCP_ECC_OPERAND_SIZE;
1646
1647 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1648 /* Copy the scalar value */
1649 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1650 ecc->u.pm.scalar_len,
1651 CCP_ECC_OPERAND_SIZE,
1652 false);
1653 if (ret)
1654 goto e_src;
1655 src.address += CCP_ECC_OPERAND_SIZE;
1656 }
1657 }
1658
1659 /* Restore the workarea address */
1660 src.address = save;
1661
1662 /* Prepare the output area for the operation */
1663 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1664 DMA_FROM_DEVICE);
1665 if (ret)
1666 goto e_src;
1667
1668 op.soc = 1;
1669 op.src.u.dma.address = src.dma.address;
1670 op.src.u.dma.offset = 0;
1671 op.src.u.dma.length = src.length;
1672 op.dst.u.dma.address = dst.dma.address;
1673 op.dst.u.dma.offset = 0;
1674 op.dst.u.dma.length = dst.length;
1675
1676 op.u.ecc.function = cmd->u.ecc.function;
1677
1678 ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
1679 if (ret) {
1680 cmd->engine_error = cmd_q->cmd_error;
1681 goto e_dst;
1682 }
1683
1684 ecc->ecc_result = le16_to_cpup(
1685 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1686 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1687 ret = -EIO;
1688 goto e_dst;
1689 }
1690
1691 /* Save the workarea address since it is updated as we walk through
1692 * to copy the point math result
1693 */
1694 save = dst.address;
1695
1696 /* Save the ECC result X and Y coordinates */
1697 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1698 CCP_ECC_MODULUS_BYTES);
1699 dst.address += CCP_ECC_OUTPUT_SIZE;
1700 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1701 CCP_ECC_MODULUS_BYTES);
1702 dst.address += CCP_ECC_OUTPUT_SIZE;
1703
1704 /* Restore the workarea address */
1705 dst.address = save;
1706
1707e_dst:
1708 ccp_dm_free(&dst);
1709
1710e_src:
1711 ccp_dm_free(&src);
1712
1713 return ret;
1714}
1715
1716static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1717{
1718 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1719
1720 ecc->ecc_result = 0;
1721
1722 if (!ecc->mod ||
1723 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1724 return -EINVAL;
1725
1726 switch (ecc->function) {
1727 case CCP_ECC_FUNCTION_MMUL_384BIT:
1728 case CCP_ECC_FUNCTION_MADD_384BIT:
1729 case CCP_ECC_FUNCTION_MINV_384BIT:
1730 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1731
1732 case CCP_ECC_FUNCTION_PADD_384BIT:
1733 case CCP_ECC_FUNCTION_PMUL_384BIT:
1734 case CCP_ECC_FUNCTION_PDBL_384BIT:
1735 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1736
1737 default:
1738 return -EINVAL;
1739 }
1740}
1741
1742int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1743{
1744 int ret;
1745
1746 cmd->engine_error = 0;
1747 cmd_q->cmd_error = 0;
1748 cmd_q->int_rcvd = 0;
1749 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
1750
1751 switch (cmd->engine) {
1752 case CCP_ENGINE_AES:
1753 ret = ccp_run_aes_cmd(cmd_q, cmd);
1754 break;
1755 case CCP_ENGINE_XTS_AES_128:
1756 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1757 break;
1758 case CCP_ENGINE_SHA:
1759 ret = ccp_run_sha_cmd(cmd_q, cmd);
1760 break;
1761 case CCP_ENGINE_RSA:
1762 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1763 break;
1764 case CCP_ENGINE_PASSTHRU:
1765 ret = ccp_run_passthru_cmd(cmd_q, cmd);
1766 break;
1767 case CCP_ENGINE_ECC:
1768 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1769 break;
1770 default:
1771 ret = -EINVAL;
1772 }
1773
1774 return ret;
1775}
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/module.h>
15#include <linux/kernel.h>
16#include <linux/pci.h>
17#include <linux/interrupt.h>
18#include <crypto/scatterwalk.h>
19#include <linux/ccp.h>
20
21#include "ccp-dev.h"
22
23/* SHA initial context values */
24static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
25 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
26 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
27 cpu_to_be32(SHA1_H4),
28};
29
30static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
31 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
32 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
33 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
34 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
35};
36
37static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
38 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
39 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
40 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
41 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
42};
43
44#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
45 ccp_gen_jobid(ccp) : 0)
46
47static u32 ccp_gen_jobid(struct ccp_device *ccp)
48{
49 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
50}
51
52static void ccp_sg_free(struct ccp_sg_workarea *wa)
53{
54 if (wa->dma_count)
55 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
56
57 wa->dma_count = 0;
58}
59
60static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
61 struct scatterlist *sg, u64 len,
62 enum dma_data_direction dma_dir)
63{
64 memset(wa, 0, sizeof(*wa));
65
66 wa->sg = sg;
67 if (!sg)
68 return 0;
69
70 wa->nents = sg_nents_for_len(sg, len);
71 if (wa->nents < 0)
72 return wa->nents;
73
74 wa->bytes_left = len;
75 wa->sg_used = 0;
76
77 if (len == 0)
78 return 0;
79
80 if (dma_dir == DMA_NONE)
81 return 0;
82
83 wa->dma_sg = sg;
84 wa->dma_dev = dev;
85 wa->dma_dir = dma_dir;
86 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
87 if (!wa->dma_count)
88 return -ENOMEM;
89
90 return 0;
91}
92
93static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
94{
95 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
96
97 if (!wa->sg)
98 return;
99
100 wa->sg_used += nbytes;
101 wa->bytes_left -= nbytes;
102 if (wa->sg_used == wa->sg->length) {
103 wa->sg = sg_next(wa->sg);
104 wa->sg_used = 0;
105 }
106}
107
108static void ccp_dm_free(struct ccp_dm_workarea *wa)
109{
110 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
111 if (wa->address)
112 dma_pool_free(wa->dma_pool, wa->address,
113 wa->dma.address);
114 } else {
115 if (wa->dma.address)
116 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
117 wa->dma.dir);
118 kfree(wa->address);
119 }
120
121 wa->address = NULL;
122 wa->dma.address = 0;
123}
124
125static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
126 struct ccp_cmd_queue *cmd_q,
127 unsigned int len,
128 enum dma_data_direction dir)
129{
130 memset(wa, 0, sizeof(*wa));
131
132 if (!len)
133 return 0;
134
135 wa->dev = cmd_q->ccp->dev;
136 wa->length = len;
137
138 if (len <= CCP_DMAPOOL_MAX_SIZE) {
139 wa->dma_pool = cmd_q->dma_pool;
140
141 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
142 &wa->dma.address);
143 if (!wa->address)
144 return -ENOMEM;
145
146 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
147
148 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
149 } else {
150 wa->address = kzalloc(len, GFP_KERNEL);
151 if (!wa->address)
152 return -ENOMEM;
153
154 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
155 dir);
156 if (!wa->dma.address)
157 return -ENOMEM;
158
159 wa->dma.length = len;
160 }
161 wa->dma.dir = dir;
162
163 return 0;
164}
165
166static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
167 struct scatterlist *sg, unsigned int sg_offset,
168 unsigned int len)
169{
170 WARN_ON(!wa->address);
171
172 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
173 0);
174}
175
176static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
177 struct scatterlist *sg, unsigned int sg_offset,
178 unsigned int len)
179{
180 WARN_ON(!wa->address);
181
182 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
183 1);
184}
185
186static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
187 struct scatterlist *sg,
188 unsigned int len, unsigned int se_len,
189 bool sign_extend)
190{
191 unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
192 u8 buffer[CCP_REVERSE_BUF_SIZE];
193
194 if (WARN_ON(se_len > sizeof(buffer)))
195 return -EINVAL;
196
197 sg_offset = len;
198 dm_offset = 0;
199 nbytes = len;
200 while (nbytes) {
201 sb_len = min_t(unsigned int, nbytes, se_len);
202 sg_offset -= sb_len;
203
204 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
205 for (i = 0; i < sb_len; i++)
206 wa->address[dm_offset + i] = buffer[sb_len - i - 1];
207
208 dm_offset += sb_len;
209 nbytes -= sb_len;
210
211 if ((sb_len != se_len) && sign_extend) {
212 /* Must sign-extend to nearest sign-extend length */
213 if (wa->address[dm_offset - 1] & 0x80)
214 memset(wa->address + dm_offset, 0xff,
215 se_len - sb_len);
216 }
217 }
218
219 return 0;
220}
221
222static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
223 struct scatterlist *sg,
224 unsigned int len)
225{
226 unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
227 u8 buffer[CCP_REVERSE_BUF_SIZE];
228
229 sg_offset = 0;
230 dm_offset = len;
231 nbytes = len;
232 while (nbytes) {
233 sb_len = min_t(unsigned int, nbytes, sizeof(buffer));
234 dm_offset -= sb_len;
235
236 for (i = 0; i < sb_len; i++)
237 buffer[sb_len - i - 1] = wa->address[dm_offset + i];
238 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
239
240 sg_offset += sb_len;
241 nbytes -= sb_len;
242 }
243}
244
245static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
246{
247 ccp_dm_free(&data->dm_wa);
248 ccp_sg_free(&data->sg_wa);
249}
250
251static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
252 struct scatterlist *sg, u64 sg_len,
253 unsigned int dm_len,
254 enum dma_data_direction dir)
255{
256 int ret;
257
258 memset(data, 0, sizeof(*data));
259
260 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
261 dir);
262 if (ret)
263 goto e_err;
264
265 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
266 if (ret)
267 goto e_err;
268
269 return 0;
270
271e_err:
272 ccp_free_data(data, cmd_q);
273
274 return ret;
275}
276
277static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
278{
279 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
280 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
281 unsigned int buf_count, nbytes;
282
283 /* Clear the buffer if setting it */
284 if (!from)
285 memset(dm_wa->address, 0, dm_wa->length);
286
287 if (!sg_wa->sg)
288 return 0;
289
290 /* Perform the copy operation
291 * nbytes will always be <= UINT_MAX because dm_wa->length is
292 * an unsigned int
293 */
294 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
295 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
296 nbytes, from);
297
298 /* Update the structures and generate the count */
299 buf_count = 0;
300 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
301 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
302 dm_wa->length - buf_count);
303 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
304
305 buf_count += nbytes;
306 ccp_update_sg_workarea(sg_wa, nbytes);
307 }
308
309 return buf_count;
310}
311
312static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
313{
314 return ccp_queue_buf(data, 0);
315}
316
317static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
318{
319 return ccp_queue_buf(data, 1);
320}
321
322static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
323 struct ccp_op *op, unsigned int block_size,
324 bool blocksize_op)
325{
326 unsigned int sg_src_len, sg_dst_len, op_len;
327
328 /* The CCP can only DMA from/to one address each per operation. This
329 * requires that we find the smallest DMA area between the source
330 * and destination. The resulting len values will always be <= UINT_MAX
331 * because the dma length is an unsigned int.
332 */
333 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
334 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
335
336 if (dst) {
337 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
338 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
339 op_len = min(sg_src_len, sg_dst_len);
340 } else {
341 op_len = sg_src_len;
342 }
343
344 /* The data operation length will be at least block_size in length
345 * or the smaller of available sg room remaining for the source or
346 * the destination
347 */
348 op_len = max(op_len, block_size);
349
350 /* Unless we have to buffer data, there's no reason to wait */
351 op->soc = 0;
352
353 if (sg_src_len < block_size) {
354 /* Not enough data in the sg element, so it
355 * needs to be buffered into a blocksize chunk
356 */
357 int cp_len = ccp_fill_queue_buf(src);
358
359 op->soc = 1;
360 op->src.u.dma.address = src->dm_wa.dma.address;
361 op->src.u.dma.offset = 0;
362 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
363 } else {
364 /* Enough data in the sg element, but we need to
365 * adjust for any previously copied data
366 */
367 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
368 op->src.u.dma.offset = src->sg_wa.sg_used;
369 op->src.u.dma.length = op_len & ~(block_size - 1);
370
371 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
372 }
373
374 if (dst) {
375 if (sg_dst_len < block_size) {
376 /* Not enough room in the sg element or we're on the
377 * last piece of data (when using padding), so the
378 * output needs to be buffered into a blocksize chunk
379 */
380 op->soc = 1;
381 op->dst.u.dma.address = dst->dm_wa.dma.address;
382 op->dst.u.dma.offset = 0;
383 op->dst.u.dma.length = op->src.u.dma.length;
384 } else {
385 /* Enough room in the sg element, but we need to
386 * adjust for any previously used area
387 */
388 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
389 op->dst.u.dma.offset = dst->sg_wa.sg_used;
390 op->dst.u.dma.length = op->src.u.dma.length;
391 }
392 }
393}
394
395static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
396 struct ccp_op *op)
397{
398 op->init = 0;
399
400 if (dst) {
401 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
402 ccp_empty_queue_buf(dst);
403 else
404 ccp_update_sg_workarea(&dst->sg_wa,
405 op->dst.u.dma.length);
406 }
407}
408
409static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
410 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
411 u32 byte_swap, bool from)
412{
413 struct ccp_op op;
414
415 memset(&op, 0, sizeof(op));
416
417 op.cmd_q = cmd_q;
418 op.jobid = jobid;
419 op.eom = 1;
420
421 if (from) {
422 op.soc = 1;
423 op.src.type = CCP_MEMTYPE_SB;
424 op.src.u.sb = sb;
425 op.dst.type = CCP_MEMTYPE_SYSTEM;
426 op.dst.u.dma.address = wa->dma.address;
427 op.dst.u.dma.length = wa->length;
428 } else {
429 op.src.type = CCP_MEMTYPE_SYSTEM;
430 op.src.u.dma.address = wa->dma.address;
431 op.src.u.dma.length = wa->length;
432 op.dst.type = CCP_MEMTYPE_SB;
433 op.dst.u.sb = sb;
434 }
435
436 op.u.passthru.byte_swap = byte_swap;
437
438 return cmd_q->ccp->vdata->perform->passthru(&op);
439}
440
441static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
442 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
443 u32 byte_swap)
444{
445 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
446}
447
448static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
449 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
450 u32 byte_swap)
451{
452 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
453}
454
455static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
456 struct ccp_cmd *cmd)
457{
458 struct ccp_aes_engine *aes = &cmd->u.aes;
459 struct ccp_dm_workarea key, ctx;
460 struct ccp_data src;
461 struct ccp_op op;
462 unsigned int dm_offset;
463 int ret;
464
465 if (!((aes->key_len == AES_KEYSIZE_128) ||
466 (aes->key_len == AES_KEYSIZE_192) ||
467 (aes->key_len == AES_KEYSIZE_256)))
468 return -EINVAL;
469
470 if (aes->src_len & (AES_BLOCK_SIZE - 1))
471 return -EINVAL;
472
473 if (aes->iv_len != AES_BLOCK_SIZE)
474 return -EINVAL;
475
476 if (!aes->key || !aes->iv || !aes->src)
477 return -EINVAL;
478
479 if (aes->cmac_final) {
480 if (aes->cmac_key_len != AES_BLOCK_SIZE)
481 return -EINVAL;
482
483 if (!aes->cmac_key)
484 return -EINVAL;
485 }
486
487 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
488 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
489
490 ret = -EIO;
491 memset(&op, 0, sizeof(op));
492 op.cmd_q = cmd_q;
493 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
494 op.sb_key = cmd_q->sb_key;
495 op.sb_ctx = cmd_q->sb_ctx;
496 op.init = 1;
497 op.u.aes.type = aes->type;
498 op.u.aes.mode = aes->mode;
499 op.u.aes.action = aes->action;
500
501 /* All supported key sizes fit in a single (32-byte) SB entry
502 * and must be in little endian format. Use the 256-bit byte
503 * swap passthru option to convert from big endian to little
504 * endian.
505 */
506 ret = ccp_init_dm_workarea(&key, cmd_q,
507 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
508 DMA_TO_DEVICE);
509 if (ret)
510 return ret;
511
512 dm_offset = CCP_SB_BYTES - aes->key_len;
513 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
514 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
515 CCP_PASSTHRU_BYTESWAP_256BIT);
516 if (ret) {
517 cmd->engine_error = cmd_q->cmd_error;
518 goto e_key;
519 }
520
521 /* The AES context fits in a single (32-byte) SB entry and
522 * must be in little endian format. Use the 256-bit byte swap
523 * passthru option to convert from big endian to little endian.
524 */
525 ret = ccp_init_dm_workarea(&ctx, cmd_q,
526 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
527 DMA_BIDIRECTIONAL);
528 if (ret)
529 goto e_key;
530
531 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
532 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
533 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
534 CCP_PASSTHRU_BYTESWAP_256BIT);
535 if (ret) {
536 cmd->engine_error = cmd_q->cmd_error;
537 goto e_ctx;
538 }
539
540 /* Send data to the CCP AES engine */
541 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
542 AES_BLOCK_SIZE, DMA_TO_DEVICE);
543 if (ret)
544 goto e_ctx;
545
546 while (src.sg_wa.bytes_left) {
547 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
548 if (aes->cmac_final && !src.sg_wa.bytes_left) {
549 op.eom = 1;
550
551 /* Push the K1/K2 key to the CCP now */
552 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
553 op.sb_ctx,
554 CCP_PASSTHRU_BYTESWAP_256BIT);
555 if (ret) {
556 cmd->engine_error = cmd_q->cmd_error;
557 goto e_src;
558 }
559
560 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
561 aes->cmac_key_len);
562 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
563 CCP_PASSTHRU_BYTESWAP_256BIT);
564 if (ret) {
565 cmd->engine_error = cmd_q->cmd_error;
566 goto e_src;
567 }
568 }
569
570 ret = cmd_q->ccp->vdata->perform->aes(&op);
571 if (ret) {
572 cmd->engine_error = cmd_q->cmd_error;
573 goto e_src;
574 }
575
576 ccp_process_data(&src, NULL, &op);
577 }
578
579 /* Retrieve the AES context - convert from LE to BE using
580 * 32-byte (256-bit) byteswapping
581 */
582 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
583 CCP_PASSTHRU_BYTESWAP_256BIT);
584 if (ret) {
585 cmd->engine_error = cmd_q->cmd_error;
586 goto e_src;
587 }
588
589 /* ...but we only need AES_BLOCK_SIZE bytes */
590 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
591 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
592
593e_src:
594 ccp_free_data(&src, cmd_q);
595
596e_ctx:
597 ccp_dm_free(&ctx);
598
599e_key:
600 ccp_dm_free(&key);
601
602 return ret;
603}
604
605static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
606{
607 struct ccp_aes_engine *aes = &cmd->u.aes;
608 struct ccp_dm_workarea key, ctx;
609 struct ccp_data src, dst;
610 struct ccp_op op;
611 unsigned int dm_offset;
612 bool in_place = false;
613 int ret;
614
615 if (aes->mode == CCP_AES_MODE_CMAC)
616 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
617
618 if (!((aes->key_len == AES_KEYSIZE_128) ||
619 (aes->key_len == AES_KEYSIZE_192) ||
620 (aes->key_len == AES_KEYSIZE_256)))
621 return -EINVAL;
622
623 if (((aes->mode == CCP_AES_MODE_ECB) ||
624 (aes->mode == CCP_AES_MODE_CBC) ||
625 (aes->mode == CCP_AES_MODE_CFB)) &&
626 (aes->src_len & (AES_BLOCK_SIZE - 1)))
627 return -EINVAL;
628
629 if (!aes->key || !aes->src || !aes->dst)
630 return -EINVAL;
631
632 if (aes->mode != CCP_AES_MODE_ECB) {
633 if (aes->iv_len != AES_BLOCK_SIZE)
634 return -EINVAL;
635
636 if (!aes->iv)
637 return -EINVAL;
638 }
639
640 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
641 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
642
643 ret = -EIO;
644 memset(&op, 0, sizeof(op));
645 op.cmd_q = cmd_q;
646 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
647 op.sb_key = cmd_q->sb_key;
648 op.sb_ctx = cmd_q->sb_ctx;
649 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
650 op.u.aes.type = aes->type;
651 op.u.aes.mode = aes->mode;
652 op.u.aes.action = aes->action;
653
654 /* All supported key sizes fit in a single (32-byte) SB entry
655 * and must be in little endian format. Use the 256-bit byte
656 * swap passthru option to convert from big endian to little
657 * endian.
658 */
659 ret = ccp_init_dm_workarea(&key, cmd_q,
660 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
661 DMA_TO_DEVICE);
662 if (ret)
663 return ret;
664
665 dm_offset = CCP_SB_BYTES - aes->key_len;
666 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
667 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
668 CCP_PASSTHRU_BYTESWAP_256BIT);
669 if (ret) {
670 cmd->engine_error = cmd_q->cmd_error;
671 goto e_key;
672 }
673
674 /* The AES context fits in a single (32-byte) SB entry and
675 * must be in little endian format. Use the 256-bit byte swap
676 * passthru option to convert from big endian to little endian.
677 */
678 ret = ccp_init_dm_workarea(&ctx, cmd_q,
679 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
680 DMA_BIDIRECTIONAL);
681 if (ret)
682 goto e_key;
683
684 if (aes->mode != CCP_AES_MODE_ECB) {
685 /* Load the AES context - convert to LE */
686 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
687 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
688 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
689 CCP_PASSTHRU_BYTESWAP_256BIT);
690 if (ret) {
691 cmd->engine_error = cmd_q->cmd_error;
692 goto e_ctx;
693 }
694 }
695
696 /* Prepare the input and output data workareas. For in-place
697 * operations we need to set the dma direction to BIDIRECTIONAL
698 * and copy the src workarea to the dst workarea.
699 */
700 if (sg_virt(aes->src) == sg_virt(aes->dst))
701 in_place = true;
702
703 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
704 AES_BLOCK_SIZE,
705 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
706 if (ret)
707 goto e_ctx;
708
709 if (in_place) {
710 dst = src;
711 } else {
712 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
713 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
714 if (ret)
715 goto e_src;
716 }
717
718 /* Send data to the CCP AES engine */
719 while (src.sg_wa.bytes_left) {
720 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
721 if (!src.sg_wa.bytes_left) {
722 op.eom = 1;
723
724 /* Since we don't retrieve the AES context in ECB
725 * mode we have to wait for the operation to complete
726 * on the last piece of data
727 */
728 if (aes->mode == CCP_AES_MODE_ECB)
729 op.soc = 1;
730 }
731
732 ret = cmd_q->ccp->vdata->perform->aes(&op);
733 if (ret) {
734 cmd->engine_error = cmd_q->cmd_error;
735 goto e_dst;
736 }
737
738 ccp_process_data(&src, &dst, &op);
739 }
740
741 if (aes->mode != CCP_AES_MODE_ECB) {
742 /* Retrieve the AES context - convert from LE to BE using
743 * 32-byte (256-bit) byteswapping
744 */
745 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
746 CCP_PASSTHRU_BYTESWAP_256BIT);
747 if (ret) {
748 cmd->engine_error = cmd_q->cmd_error;
749 goto e_dst;
750 }
751
752 /* ...but we only need AES_BLOCK_SIZE bytes */
753 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
754 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
755 }
756
757e_dst:
758 if (!in_place)
759 ccp_free_data(&dst, cmd_q);
760
761e_src:
762 ccp_free_data(&src, cmd_q);
763
764e_ctx:
765 ccp_dm_free(&ctx);
766
767e_key:
768 ccp_dm_free(&key);
769
770 return ret;
771}
772
773static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
774 struct ccp_cmd *cmd)
775{
776 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
777 struct ccp_dm_workarea key, ctx;
778 struct ccp_data src, dst;
779 struct ccp_op op;
780 unsigned int unit_size, dm_offset;
781 bool in_place = false;
782 int ret;
783
784 switch (xts->unit_size) {
785 case CCP_XTS_AES_UNIT_SIZE_16:
786 unit_size = 16;
787 break;
788 case CCP_XTS_AES_UNIT_SIZE_512:
789 unit_size = 512;
790 break;
791 case CCP_XTS_AES_UNIT_SIZE_1024:
792 unit_size = 1024;
793 break;
794 case CCP_XTS_AES_UNIT_SIZE_2048:
795 unit_size = 2048;
796 break;
797 case CCP_XTS_AES_UNIT_SIZE_4096:
798 unit_size = 4096;
799 break;
800
801 default:
802 return -EINVAL;
803 }
804
805 if (xts->key_len != AES_KEYSIZE_128)
806 return -EINVAL;
807
808 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
809 return -EINVAL;
810
811 if (xts->iv_len != AES_BLOCK_SIZE)
812 return -EINVAL;
813
814 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
815 return -EINVAL;
816
817 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
818 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
819
820 ret = -EIO;
821 memset(&op, 0, sizeof(op));
822 op.cmd_q = cmd_q;
823 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
824 op.sb_key = cmd_q->sb_key;
825 op.sb_ctx = cmd_q->sb_ctx;
826 op.init = 1;
827 op.u.xts.action = xts->action;
828 op.u.xts.unit_size = xts->unit_size;
829
830 /* All supported key sizes fit in a single (32-byte) SB entry
831 * and must be in little endian format. Use the 256-bit byte
832 * swap passthru option to convert from big endian to little
833 * endian.
834 */
835 ret = ccp_init_dm_workarea(&key, cmd_q,
836 CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES,
837 DMA_TO_DEVICE);
838 if (ret)
839 return ret;
840
841 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
842 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
843 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
844 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
845 CCP_PASSTHRU_BYTESWAP_256BIT);
846 if (ret) {
847 cmd->engine_error = cmd_q->cmd_error;
848 goto e_key;
849 }
850
851 /* The AES context fits in a single (32-byte) SB entry and
852 * for XTS is already in little endian format so no byte swapping
853 * is needed.
854 */
855 ret = ccp_init_dm_workarea(&ctx, cmd_q,
856 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
857 DMA_BIDIRECTIONAL);
858 if (ret)
859 goto e_key;
860
861 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
862 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
863 CCP_PASSTHRU_BYTESWAP_NOOP);
864 if (ret) {
865 cmd->engine_error = cmd_q->cmd_error;
866 goto e_ctx;
867 }
868
869 /* Prepare the input and output data workareas. For in-place
870 * operations we need to set the dma direction to BIDIRECTIONAL
871 * and copy the src workarea to the dst workarea.
872 */
873 if (sg_virt(xts->src) == sg_virt(xts->dst))
874 in_place = true;
875
876 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
877 unit_size,
878 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
879 if (ret)
880 goto e_ctx;
881
882 if (in_place) {
883 dst = src;
884 } else {
885 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
886 unit_size, DMA_FROM_DEVICE);
887 if (ret)
888 goto e_src;
889 }
890
891 /* Send data to the CCP AES engine */
892 while (src.sg_wa.bytes_left) {
893 ccp_prepare_data(&src, &dst, &op, unit_size, true);
894 if (!src.sg_wa.bytes_left)
895 op.eom = 1;
896
897 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
898 if (ret) {
899 cmd->engine_error = cmd_q->cmd_error;
900 goto e_dst;
901 }
902
903 ccp_process_data(&src, &dst, &op);
904 }
905
906 /* Retrieve the AES context - convert from LE to BE using
907 * 32-byte (256-bit) byteswapping
908 */
909 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
910 CCP_PASSTHRU_BYTESWAP_256BIT);
911 if (ret) {
912 cmd->engine_error = cmd_q->cmd_error;
913 goto e_dst;
914 }
915
916 /* ...but we only need AES_BLOCK_SIZE bytes */
917 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
918 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
919
920e_dst:
921 if (!in_place)
922 ccp_free_data(&dst, cmd_q);
923
924e_src:
925 ccp_free_data(&src, cmd_q);
926
927e_ctx:
928 ccp_dm_free(&ctx);
929
930e_key:
931 ccp_dm_free(&key);
932
933 return ret;
934}
935
936static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
937{
938 struct ccp_sha_engine *sha = &cmd->u.sha;
939 struct ccp_dm_workarea ctx;
940 struct ccp_data src;
941 struct ccp_op op;
942 unsigned int ioffset, ooffset;
943 unsigned int digest_size;
944 int sb_count;
945 const void *init;
946 u64 block_size;
947 int ctx_size;
948 int ret;
949
950 switch (sha->type) {
951 case CCP_SHA_TYPE_1:
952 if (sha->ctx_len < SHA1_DIGEST_SIZE)
953 return -EINVAL;
954 block_size = SHA1_BLOCK_SIZE;
955 break;
956 case CCP_SHA_TYPE_224:
957 if (sha->ctx_len < SHA224_DIGEST_SIZE)
958 return -EINVAL;
959 block_size = SHA224_BLOCK_SIZE;
960 break;
961 case CCP_SHA_TYPE_256:
962 if (sha->ctx_len < SHA256_DIGEST_SIZE)
963 return -EINVAL;
964 block_size = SHA256_BLOCK_SIZE;
965 break;
966 default:
967 return -EINVAL;
968 }
969
970 if (!sha->ctx)
971 return -EINVAL;
972
973 if (!sha->final && (sha->src_len & (block_size - 1)))
974 return -EINVAL;
975
976 /* The version 3 device can't handle zero-length input */
977 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
978
979 if (!sha->src_len) {
980 unsigned int digest_len;
981 const u8 *sha_zero;
982
983 /* Not final, just return */
984 if (!sha->final)
985 return 0;
986
987 /* CCP can't do a zero length sha operation so the
988 * caller must buffer the data.
989 */
990 if (sha->msg_bits)
991 return -EINVAL;
992
993 /* The CCP cannot perform zero-length sha operations
994 * so the caller is required to buffer data for the
995 * final operation. However, a sha operation for a
996 * message with a total length of zero is valid so
997 * known values are required to supply the result.
998 */
999 switch (sha->type) {
1000 case CCP_SHA_TYPE_1:
1001 sha_zero = sha1_zero_message_hash;
1002 digest_len = SHA1_DIGEST_SIZE;
1003 break;
1004 case CCP_SHA_TYPE_224:
1005 sha_zero = sha224_zero_message_hash;
1006 digest_len = SHA224_DIGEST_SIZE;
1007 break;
1008 case CCP_SHA_TYPE_256:
1009 sha_zero = sha256_zero_message_hash;
1010 digest_len = SHA256_DIGEST_SIZE;
1011 break;
1012 default:
1013 return -EINVAL;
1014 }
1015
1016 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1017 digest_len, 1);
1018
1019 return 0;
1020 }
1021 }
1022
1023 /* Set variables used throughout */
1024 switch (sha->type) {
1025 case CCP_SHA_TYPE_1:
1026 digest_size = SHA1_DIGEST_SIZE;
1027 init = (void *) ccp_sha1_init;
1028 ctx_size = SHA1_DIGEST_SIZE;
1029 sb_count = 1;
1030 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1031 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1032 else
1033 ooffset = ioffset = 0;
1034 break;
1035 case CCP_SHA_TYPE_224:
1036 digest_size = SHA224_DIGEST_SIZE;
1037 init = (void *) ccp_sha224_init;
1038 ctx_size = SHA256_DIGEST_SIZE;
1039 sb_count = 1;
1040 ioffset = 0;
1041 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1042 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1043 else
1044 ooffset = 0;
1045 break;
1046 case CCP_SHA_TYPE_256:
1047 digest_size = SHA256_DIGEST_SIZE;
1048 init = (void *) ccp_sha256_init;
1049 ctx_size = SHA256_DIGEST_SIZE;
1050 sb_count = 1;
1051 ooffset = ioffset = 0;
1052 break;
1053 default:
1054 ret = -EINVAL;
1055 goto e_data;
1056 }
1057
1058 /* For zero-length plaintext the src pointer is ignored;
1059 * otherwise both parts must be valid
1060 */
1061 if (sha->src_len && !sha->src)
1062 return -EINVAL;
1063
1064 memset(&op, 0, sizeof(op));
1065 op.cmd_q = cmd_q;
1066 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1067 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1068 op.u.sha.type = sha->type;
1069 op.u.sha.msg_bits = sha->msg_bits;
1070
1071 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1072 DMA_BIDIRECTIONAL);
1073 if (ret)
1074 return ret;
1075 if (sha->first) {
1076 switch (sha->type) {
1077 case CCP_SHA_TYPE_1:
1078 case CCP_SHA_TYPE_224:
1079 case CCP_SHA_TYPE_256:
1080 memcpy(ctx.address + ioffset, init, ctx_size);
1081 break;
1082 default:
1083 ret = -EINVAL;
1084 goto e_ctx;
1085 }
1086 } else {
1087 /* Restore the context */
1088 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1089 sb_count * CCP_SB_BYTES);
1090 }
1091
1092 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1093 CCP_PASSTHRU_BYTESWAP_256BIT);
1094 if (ret) {
1095 cmd->engine_error = cmd_q->cmd_error;
1096 goto e_ctx;
1097 }
1098
1099 if (sha->src) {
1100 /* Send data to the CCP SHA engine; block_size is set above */
1101 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1102 block_size, DMA_TO_DEVICE);
1103 if (ret)
1104 goto e_ctx;
1105
1106 while (src.sg_wa.bytes_left) {
1107 ccp_prepare_data(&src, NULL, &op, block_size, false);
1108 if (sha->final && !src.sg_wa.bytes_left)
1109 op.eom = 1;
1110
1111 ret = cmd_q->ccp->vdata->perform->sha(&op);
1112 if (ret) {
1113 cmd->engine_error = cmd_q->cmd_error;
1114 goto e_data;
1115 }
1116
1117 ccp_process_data(&src, NULL, &op);
1118 }
1119 } else {
1120 op.eom = 1;
1121 ret = cmd_q->ccp->vdata->perform->sha(&op);
1122 if (ret) {
1123 cmd->engine_error = cmd_q->cmd_error;
1124 goto e_data;
1125 }
1126 }
1127
1128 /* Retrieve the SHA context - convert from LE to BE using
1129 * 32-byte (256-bit) byteswapping to BE
1130 */
1131 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1132 CCP_PASSTHRU_BYTESWAP_256BIT);
1133 if (ret) {
1134 cmd->engine_error = cmd_q->cmd_error;
1135 goto e_data;
1136 }
1137
1138 if (sha->final) {
1139 /* Finishing up, so get the digest */
1140 switch (sha->type) {
1141 case CCP_SHA_TYPE_1:
1142 case CCP_SHA_TYPE_224:
1143 case CCP_SHA_TYPE_256:
1144 ccp_get_dm_area(&ctx, ooffset,
1145 sha->ctx, 0,
1146 digest_size);
1147 break;
1148 default:
1149 ret = -EINVAL;
1150 goto e_ctx;
1151 }
1152 } else {
1153 /* Stash the context */
1154 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1155 sb_count * CCP_SB_BYTES);
1156 }
1157
1158 if (sha->final && sha->opad) {
1159 /* HMAC operation, recursively perform final SHA */
1160 struct ccp_cmd hmac_cmd;
1161 struct scatterlist sg;
1162 u8 *hmac_buf;
1163
1164 if (sha->opad_len != block_size) {
1165 ret = -EINVAL;
1166 goto e_data;
1167 }
1168
1169 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1170 if (!hmac_buf) {
1171 ret = -ENOMEM;
1172 goto e_data;
1173 }
1174 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1175
1176 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1177 switch (sha->type) {
1178 case CCP_SHA_TYPE_1:
1179 case CCP_SHA_TYPE_224:
1180 case CCP_SHA_TYPE_256:
1181 memcpy(hmac_buf + block_size,
1182 ctx.address + ooffset,
1183 digest_size);
1184 break;
1185 default:
1186 ret = -EINVAL;
1187 goto e_ctx;
1188 }
1189
1190 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1191 hmac_cmd.engine = CCP_ENGINE_SHA;
1192 hmac_cmd.u.sha.type = sha->type;
1193 hmac_cmd.u.sha.ctx = sha->ctx;
1194 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1195 hmac_cmd.u.sha.src = &sg;
1196 hmac_cmd.u.sha.src_len = block_size + digest_size;
1197 hmac_cmd.u.sha.opad = NULL;
1198 hmac_cmd.u.sha.opad_len = 0;
1199 hmac_cmd.u.sha.first = 1;
1200 hmac_cmd.u.sha.final = 1;
1201 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1202
1203 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1204 if (ret)
1205 cmd->engine_error = hmac_cmd.engine_error;
1206
1207 kfree(hmac_buf);
1208 }
1209
1210e_data:
1211 if (sha->src)
1212 ccp_free_data(&src, cmd_q);
1213
1214e_ctx:
1215 ccp_dm_free(&ctx);
1216
1217 return ret;
1218}
1219
1220static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1221{
1222 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1223 struct ccp_dm_workarea exp, src;
1224 struct ccp_data dst;
1225 struct ccp_op op;
1226 unsigned int sb_count, i_len, o_len;
1227 int ret;
1228
1229 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1230 return -EINVAL;
1231
1232 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1233 return -EINVAL;
1234
1235 /* The RSA modulus must precede the message being acted upon, so
1236 * it must be copied to a DMA area where the message and the
1237 * modulus can be concatenated. Therefore the input buffer
1238 * length required is twice the output buffer length (which
1239 * must be a multiple of 256-bits).
1240 */
1241 o_len = ((rsa->key_size + 255) / 256) * 32;
1242 i_len = o_len * 2;
1243
1244 sb_count = o_len / CCP_SB_BYTES;
1245
1246 memset(&op, 0, sizeof(op));
1247 op.cmd_q = cmd_q;
1248 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1249 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
1250
1251 if (!op.sb_key)
1252 return -EIO;
1253
1254 /* The RSA exponent may span multiple (32-byte) SB entries and must
1255 * be in little endian format. Reverse copy each 32-byte chunk
1256 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1257 * and each byte within that chunk and do not perform any byte swap
1258 * operations on the passthru operation.
1259 */
1260 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1261 if (ret)
1262 goto e_sb;
1263
1264 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1265 CCP_SB_BYTES, false);
1266 if (ret)
1267 goto e_exp;
1268 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1269 CCP_PASSTHRU_BYTESWAP_NOOP);
1270 if (ret) {
1271 cmd->engine_error = cmd_q->cmd_error;
1272 goto e_exp;
1273 }
1274
1275 /* Concatenate the modulus and the message. Both the modulus and
1276 * the operands must be in little endian format. Since the input
1277 * is in big endian format it must be converted.
1278 */
1279 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1280 if (ret)
1281 goto e_exp;
1282
1283 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1284 CCP_SB_BYTES, false);
1285 if (ret)
1286 goto e_src;
1287 src.address += o_len; /* Adjust the address for the copy operation */
1288 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1289 CCP_SB_BYTES, false);
1290 if (ret)
1291 goto e_src;
1292 src.address -= o_len; /* Reset the address to original value */
1293
1294 /* Prepare the output area for the operation */
1295 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1296 o_len, DMA_FROM_DEVICE);
1297 if (ret)
1298 goto e_src;
1299
1300 op.soc = 1;
1301 op.src.u.dma.address = src.dma.address;
1302 op.src.u.dma.offset = 0;
1303 op.src.u.dma.length = i_len;
1304 op.dst.u.dma.address = dst.dm_wa.dma.address;
1305 op.dst.u.dma.offset = 0;
1306 op.dst.u.dma.length = o_len;
1307
1308 op.u.rsa.mod_size = rsa->key_size;
1309 op.u.rsa.input_len = i_len;
1310
1311 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1312 if (ret) {
1313 cmd->engine_error = cmd_q->cmd_error;
1314 goto e_dst;
1315 }
1316
1317 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1318
1319e_dst:
1320 ccp_free_data(&dst, cmd_q);
1321
1322e_src:
1323 ccp_dm_free(&src);
1324
1325e_exp:
1326 ccp_dm_free(&exp);
1327
1328e_sb:
1329 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1330
1331 return ret;
1332}
1333
1334static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1335 struct ccp_cmd *cmd)
1336{
1337 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1338 struct ccp_dm_workarea mask;
1339 struct ccp_data src, dst;
1340 struct ccp_op op;
1341 bool in_place = false;
1342 unsigned int i;
1343 int ret = 0;
1344
1345 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1346 return -EINVAL;
1347
1348 if (!pt->src || !pt->dst)
1349 return -EINVAL;
1350
1351 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1352 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1353 return -EINVAL;
1354 if (!pt->mask)
1355 return -EINVAL;
1356 }
1357
1358 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1359
1360 memset(&op, 0, sizeof(op));
1361 op.cmd_q = cmd_q;
1362 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1363
1364 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1365 /* Load the mask */
1366 op.sb_key = cmd_q->sb_key;
1367
1368 ret = ccp_init_dm_workarea(&mask, cmd_q,
1369 CCP_PASSTHRU_SB_COUNT *
1370 CCP_SB_BYTES,
1371 DMA_TO_DEVICE);
1372 if (ret)
1373 return ret;
1374
1375 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1376 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1377 CCP_PASSTHRU_BYTESWAP_NOOP);
1378 if (ret) {
1379 cmd->engine_error = cmd_q->cmd_error;
1380 goto e_mask;
1381 }
1382 }
1383
1384 /* Prepare the input and output data workareas. For in-place
1385 * operations we need to set the dma direction to BIDIRECTIONAL
1386 * and copy the src workarea to the dst workarea.
1387 */
1388 if (sg_virt(pt->src) == sg_virt(pt->dst))
1389 in_place = true;
1390
1391 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1392 CCP_PASSTHRU_MASKSIZE,
1393 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1394 if (ret)
1395 goto e_mask;
1396
1397 if (in_place) {
1398 dst = src;
1399 } else {
1400 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1401 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1402 if (ret)
1403 goto e_src;
1404 }
1405
1406 /* Send data to the CCP Passthru engine
1407 * Because the CCP engine works on a single source and destination
1408 * dma address at a time, each entry in the source scatterlist
1409 * (after the dma_map_sg call) must be less than or equal to the
1410 * (remaining) length in the destination scatterlist entry and the
1411 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1412 */
1413 dst.sg_wa.sg_used = 0;
1414 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1415 if (!dst.sg_wa.sg ||
1416 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1417 ret = -EINVAL;
1418 goto e_dst;
1419 }
1420
1421 if (i == src.sg_wa.dma_count) {
1422 op.eom = 1;
1423 op.soc = 1;
1424 }
1425
1426 op.src.type = CCP_MEMTYPE_SYSTEM;
1427 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1428 op.src.u.dma.offset = 0;
1429 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1430
1431 op.dst.type = CCP_MEMTYPE_SYSTEM;
1432 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1433 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1434 op.dst.u.dma.length = op.src.u.dma.length;
1435
1436 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1437 if (ret) {
1438 cmd->engine_error = cmd_q->cmd_error;
1439 goto e_dst;
1440 }
1441
1442 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1443 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1444 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1445 dst.sg_wa.sg_used = 0;
1446 }
1447 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1448 }
1449
1450e_dst:
1451 if (!in_place)
1452 ccp_free_data(&dst, cmd_q);
1453
1454e_src:
1455 ccp_free_data(&src, cmd_q);
1456
1457e_mask:
1458 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1459 ccp_dm_free(&mask);
1460
1461 return ret;
1462}
1463
1464static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1465 struct ccp_cmd *cmd)
1466{
1467 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
1468 struct ccp_dm_workarea mask;
1469 struct ccp_op op;
1470 int ret;
1471
1472 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1473 return -EINVAL;
1474
1475 if (!pt->src_dma || !pt->dst_dma)
1476 return -EINVAL;
1477
1478 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1479 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1480 return -EINVAL;
1481 if (!pt->mask)
1482 return -EINVAL;
1483 }
1484
1485 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1486
1487 memset(&op, 0, sizeof(op));
1488 op.cmd_q = cmd_q;
1489 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1490
1491 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1492 /* Load the mask */
1493 op.sb_key = cmd_q->sb_key;
1494
1495 mask.length = pt->mask_len;
1496 mask.dma.address = pt->mask;
1497 mask.dma.length = pt->mask_len;
1498
1499 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1500 CCP_PASSTHRU_BYTESWAP_NOOP);
1501 if (ret) {
1502 cmd->engine_error = cmd_q->cmd_error;
1503 return ret;
1504 }
1505 }
1506
1507 /* Send data to the CCP Passthru engine */
1508 op.eom = 1;
1509 op.soc = 1;
1510
1511 op.src.type = CCP_MEMTYPE_SYSTEM;
1512 op.src.u.dma.address = pt->src_dma;
1513 op.src.u.dma.offset = 0;
1514 op.src.u.dma.length = pt->src_len;
1515
1516 op.dst.type = CCP_MEMTYPE_SYSTEM;
1517 op.dst.u.dma.address = pt->dst_dma;
1518 op.dst.u.dma.offset = 0;
1519 op.dst.u.dma.length = pt->src_len;
1520
1521 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1522 if (ret)
1523 cmd->engine_error = cmd_q->cmd_error;
1524
1525 return ret;
1526}
1527
1528static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1529{
1530 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1531 struct ccp_dm_workarea src, dst;
1532 struct ccp_op op;
1533 int ret;
1534 u8 *save;
1535
1536 if (!ecc->u.mm.operand_1 ||
1537 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1538 return -EINVAL;
1539
1540 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1541 if (!ecc->u.mm.operand_2 ||
1542 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1543 return -EINVAL;
1544
1545 if (!ecc->u.mm.result ||
1546 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1547 return -EINVAL;
1548
1549 memset(&op, 0, sizeof(op));
1550 op.cmd_q = cmd_q;
1551 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1552
1553 /* Concatenate the modulus and the operands. Both the modulus and
1554 * the operands must be in little endian format. Since the input
1555 * is in big endian format it must be converted and placed in a
1556 * fixed length buffer.
1557 */
1558 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1559 DMA_TO_DEVICE);
1560 if (ret)
1561 return ret;
1562
1563 /* Save the workarea address since it is updated in order to perform
1564 * the concatenation
1565 */
1566 save = src.address;
1567
1568 /* Copy the ECC modulus */
1569 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1570 CCP_ECC_OPERAND_SIZE, false);
1571 if (ret)
1572 goto e_src;
1573 src.address += CCP_ECC_OPERAND_SIZE;
1574
1575 /* Copy the first operand */
1576 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1577 ecc->u.mm.operand_1_len,
1578 CCP_ECC_OPERAND_SIZE, false);
1579 if (ret)
1580 goto e_src;
1581 src.address += CCP_ECC_OPERAND_SIZE;
1582
1583 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1584 /* Copy the second operand */
1585 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1586 ecc->u.mm.operand_2_len,
1587 CCP_ECC_OPERAND_SIZE, false);
1588 if (ret)
1589 goto e_src;
1590 src.address += CCP_ECC_OPERAND_SIZE;
1591 }
1592
1593 /* Restore the workarea address */
1594 src.address = save;
1595
1596 /* Prepare the output area for the operation */
1597 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1598 DMA_FROM_DEVICE);
1599 if (ret)
1600 goto e_src;
1601
1602 op.soc = 1;
1603 op.src.u.dma.address = src.dma.address;
1604 op.src.u.dma.offset = 0;
1605 op.src.u.dma.length = src.length;
1606 op.dst.u.dma.address = dst.dma.address;
1607 op.dst.u.dma.offset = 0;
1608 op.dst.u.dma.length = dst.length;
1609
1610 op.u.ecc.function = cmd->u.ecc.function;
1611
1612 ret = cmd_q->ccp->vdata->perform->ecc(&op);
1613 if (ret) {
1614 cmd->engine_error = cmd_q->cmd_error;
1615 goto e_dst;
1616 }
1617
1618 ecc->ecc_result = le16_to_cpup(
1619 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1620 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1621 ret = -EIO;
1622 goto e_dst;
1623 }
1624
1625 /* Save the ECC result */
1626 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1627
1628e_dst:
1629 ccp_dm_free(&dst);
1630
1631e_src:
1632 ccp_dm_free(&src);
1633
1634 return ret;
1635}
1636
1637static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1638{
1639 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1640 struct ccp_dm_workarea src, dst;
1641 struct ccp_op op;
1642 int ret;
1643 u8 *save;
1644
1645 if (!ecc->u.pm.point_1.x ||
1646 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1647 !ecc->u.pm.point_1.y ||
1648 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1649 return -EINVAL;
1650
1651 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1652 if (!ecc->u.pm.point_2.x ||
1653 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1654 !ecc->u.pm.point_2.y ||
1655 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1656 return -EINVAL;
1657 } else {
1658 if (!ecc->u.pm.domain_a ||
1659 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1660 return -EINVAL;
1661
1662 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1663 if (!ecc->u.pm.scalar ||
1664 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1665 return -EINVAL;
1666 }
1667
1668 if (!ecc->u.pm.result.x ||
1669 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1670 !ecc->u.pm.result.y ||
1671 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1672 return -EINVAL;
1673
1674 memset(&op, 0, sizeof(op));
1675 op.cmd_q = cmd_q;
1676 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1677
1678 /* Concatenate the modulus and the operands. Both the modulus and
1679 * the operands must be in little endian format. Since the input
1680 * is in big endian format it must be converted and placed in a
1681 * fixed length buffer.
1682 */
1683 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1684 DMA_TO_DEVICE);
1685 if (ret)
1686 return ret;
1687
1688 /* Save the workarea address since it is updated in order to perform
1689 * the concatenation
1690 */
1691 save = src.address;
1692
1693 /* Copy the ECC modulus */
1694 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1695 CCP_ECC_OPERAND_SIZE, false);
1696 if (ret)
1697 goto e_src;
1698 src.address += CCP_ECC_OPERAND_SIZE;
1699
1700 /* Copy the first point X and Y coordinate */
1701 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1702 ecc->u.pm.point_1.x_len,
1703 CCP_ECC_OPERAND_SIZE, false);
1704 if (ret)
1705 goto e_src;
1706 src.address += CCP_ECC_OPERAND_SIZE;
1707 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1708 ecc->u.pm.point_1.y_len,
1709 CCP_ECC_OPERAND_SIZE, false);
1710 if (ret)
1711 goto e_src;
1712 src.address += CCP_ECC_OPERAND_SIZE;
1713
1714 /* Set the first point Z coordinate to 1 */
1715 *src.address = 0x01;
1716 src.address += CCP_ECC_OPERAND_SIZE;
1717
1718 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1719 /* Copy the second point X and Y coordinate */
1720 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1721 ecc->u.pm.point_2.x_len,
1722 CCP_ECC_OPERAND_SIZE, false);
1723 if (ret)
1724 goto e_src;
1725 src.address += CCP_ECC_OPERAND_SIZE;
1726 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1727 ecc->u.pm.point_2.y_len,
1728 CCP_ECC_OPERAND_SIZE, false);
1729 if (ret)
1730 goto e_src;
1731 src.address += CCP_ECC_OPERAND_SIZE;
1732
1733 /* Set the second point Z coordinate to 1 */
1734 *src.address = 0x01;
1735 src.address += CCP_ECC_OPERAND_SIZE;
1736 } else {
1737 /* Copy the Domain "a" parameter */
1738 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1739 ecc->u.pm.domain_a_len,
1740 CCP_ECC_OPERAND_SIZE, false);
1741 if (ret)
1742 goto e_src;
1743 src.address += CCP_ECC_OPERAND_SIZE;
1744
1745 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1746 /* Copy the scalar value */
1747 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1748 ecc->u.pm.scalar_len,
1749 CCP_ECC_OPERAND_SIZE,
1750 false);
1751 if (ret)
1752 goto e_src;
1753 src.address += CCP_ECC_OPERAND_SIZE;
1754 }
1755 }
1756
1757 /* Restore the workarea address */
1758 src.address = save;
1759
1760 /* Prepare the output area for the operation */
1761 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1762 DMA_FROM_DEVICE);
1763 if (ret)
1764 goto e_src;
1765
1766 op.soc = 1;
1767 op.src.u.dma.address = src.dma.address;
1768 op.src.u.dma.offset = 0;
1769 op.src.u.dma.length = src.length;
1770 op.dst.u.dma.address = dst.dma.address;
1771 op.dst.u.dma.offset = 0;
1772 op.dst.u.dma.length = dst.length;
1773
1774 op.u.ecc.function = cmd->u.ecc.function;
1775
1776 ret = cmd_q->ccp->vdata->perform->ecc(&op);
1777 if (ret) {
1778 cmd->engine_error = cmd_q->cmd_error;
1779 goto e_dst;
1780 }
1781
1782 ecc->ecc_result = le16_to_cpup(
1783 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1784 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1785 ret = -EIO;
1786 goto e_dst;
1787 }
1788
1789 /* Save the workarea address since it is updated as we walk through
1790 * to copy the point math result
1791 */
1792 save = dst.address;
1793
1794 /* Save the ECC result X and Y coordinates */
1795 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1796 CCP_ECC_MODULUS_BYTES);
1797 dst.address += CCP_ECC_OUTPUT_SIZE;
1798 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1799 CCP_ECC_MODULUS_BYTES);
1800 dst.address += CCP_ECC_OUTPUT_SIZE;
1801
1802 /* Restore the workarea address */
1803 dst.address = save;
1804
1805e_dst:
1806 ccp_dm_free(&dst);
1807
1808e_src:
1809 ccp_dm_free(&src);
1810
1811 return ret;
1812}
1813
1814static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1815{
1816 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1817
1818 ecc->ecc_result = 0;
1819
1820 if (!ecc->mod ||
1821 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1822 return -EINVAL;
1823
1824 switch (ecc->function) {
1825 case CCP_ECC_FUNCTION_MMUL_384BIT:
1826 case CCP_ECC_FUNCTION_MADD_384BIT:
1827 case CCP_ECC_FUNCTION_MINV_384BIT:
1828 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1829
1830 case CCP_ECC_FUNCTION_PADD_384BIT:
1831 case CCP_ECC_FUNCTION_PMUL_384BIT:
1832 case CCP_ECC_FUNCTION_PDBL_384BIT:
1833 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1834
1835 default:
1836 return -EINVAL;
1837 }
1838}
1839
1840int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1841{
1842 int ret;
1843
1844 cmd->engine_error = 0;
1845 cmd_q->cmd_error = 0;
1846 cmd_q->int_rcvd = 0;
1847 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
1848
1849 switch (cmd->engine) {
1850 case CCP_ENGINE_AES:
1851 ret = ccp_run_aes_cmd(cmd_q, cmd);
1852 break;
1853 case CCP_ENGINE_XTS_AES_128:
1854 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1855 break;
1856 case CCP_ENGINE_SHA:
1857 ret = ccp_run_sha_cmd(cmd_q, cmd);
1858 break;
1859 case CCP_ENGINE_RSA:
1860 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1861 break;
1862 case CCP_ENGINE_PASSTHRU:
1863 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
1864 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
1865 else
1866 ret = ccp_run_passthru_cmd(cmd_q, cmd);
1867 break;
1868 case CCP_ENGINE_ECC:
1869 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1870 break;
1871 default:
1872 ret = -EINVAL;
1873 }
1874
1875 return ret;
1876}