Loading...
Note: File does not exist in v6.2.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel Keem Bay OCS AES Crypto Driver.
4 *
5 * Copyright (C) 2018-2020 Intel Corporation
6 */
7
8#include <crypto/aes.h>
9#include <crypto/engine.h>
10#include <crypto/gcm.h>
11#include <crypto/internal/aead.h>
12#include <crypto/internal/skcipher.h>
13#include <crypto/scatterwalk.h>
14#include <linux/clk.h>
15#include <linux/completion.h>
16#include <linux/dma-mapping.h>
17#include <linux/err.h>
18#include <linux/interrupt.h>
19#include <linux/io.h>
20#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/of.h>
23#include <linux/platform_device.h>
24#include <linux/string.h>
25
26#include "ocs-aes.h"
27
28#define KMB_OCS_PRIORITY 350
29#define DRV_NAME "keembay-ocs-aes"
30
31#define OCS_AES_MIN_KEY_SIZE 16
32#define OCS_AES_MAX_KEY_SIZE 32
33#define OCS_AES_KEYSIZE_128 16
34#define OCS_AES_KEYSIZE_192 24
35#define OCS_AES_KEYSIZE_256 32
36#define OCS_SM4_KEY_SIZE 16
37
38/**
39 * struct ocs_aes_tctx - OCS AES Transform context
40 * @aes_dev: The OCS AES device.
41 * @key: AES/SM4 key.
42 * @key_len: The length (in bytes) of @key.
43 * @cipher: OCS cipher to use (either AES or SM4).
44 * @sw_cipher: The cipher to use as fallback.
45 * @use_fallback: Whether or not fallback cipher should be used.
46 */
47struct ocs_aes_tctx {
48 struct ocs_aes_dev *aes_dev;
49 u8 key[OCS_AES_KEYSIZE_256];
50 unsigned int key_len;
51 enum ocs_cipher cipher;
52 union {
53 struct crypto_sync_skcipher *sk;
54 struct crypto_aead *aead;
55 } sw_cipher;
56 bool use_fallback;
57};
58
59/**
60 * struct ocs_aes_rctx - OCS AES Request context.
61 * @instruction: Instruction to be executed (encrypt / decrypt).
62 * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS)
63 * @src_nents: Number of source SG entries.
64 * @dst_nents: Number of destination SG entries.
65 * @src_dma_count: The number of DMA-mapped entries of the source SG.
66 * @dst_dma_count: The number of DMA-mapped entries of the destination SG.
67 * @in_place: Whether or not this is an in place request, i.e.,
68 * src_sg == dst_sg.
69 * @src_dll: OCS DMA linked list for input data.
70 * @dst_dll: OCS DMA linked list for output data.
71 * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC
72 * mode).
73 * @cts_swap: Whether or not CTS swap must be performed.
74 * @aad_src_dll: OCS DMA linked list for input AAD data.
75 * @aad_dst_dll: OCS DMA linked list for output AAD data.
76 * @in_tag: Buffer to hold input encrypted tag (only used for
77 * CCM/GCM decrypt).
78 * @out_tag: Buffer to hold output encrypted / decrypted tag (only
79 * used for GCM encrypt / decrypt).
80 */
81struct ocs_aes_rctx {
82 /* Fields common across all modes. */
83 enum ocs_instruction instruction;
84 enum ocs_mode mode;
85 int src_nents;
86 int dst_nents;
87 int src_dma_count;
88 int dst_dma_count;
89 bool in_place;
90 struct ocs_dll_desc src_dll;
91 struct ocs_dll_desc dst_dll;
92
93 /* CBC specific */
94 u8 last_ct_blk[AES_BLOCK_SIZE];
95
96 /* CTS specific */
97 int cts_swap;
98
99 /* CCM/GCM specific */
100 struct ocs_dll_desc aad_src_dll;
101 struct ocs_dll_desc aad_dst_dll;
102 u8 in_tag[AES_BLOCK_SIZE];
103
104 /* GCM specific */
105 u8 out_tag[AES_BLOCK_SIZE];
106};
107
108/* Driver data. */
109struct ocs_aes_drv {
110 struct list_head dev_list;
111 spinlock_t lock; /* Protects dev_list. */
112};
113
114static struct ocs_aes_drv ocs_aes = {
115 .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list),
116 .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock),
117};
118
119static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx)
120{
121 struct ocs_aes_dev *aes_dev;
122
123 spin_lock(&ocs_aes.lock);
124
125 if (tctx->aes_dev) {
126 aes_dev = tctx->aes_dev;
127 goto exit;
128 }
129
130 /* Only a single OCS device available */
131 aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list);
132 tctx->aes_dev = aes_dev;
133
134exit:
135 spin_unlock(&ocs_aes.lock);
136
137 return aes_dev;
138}
139
140/*
141 * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual
142 * key is being passed in.
143 *
144 * Return: 0 if key is valid, -EINVAL otherwise.
145 */
146static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher)
147{
148 if (!in_key)
149 return -EINVAL;
150
151 /* For AES, only 128-byte or 256-byte keys are supported. */
152 if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 ||
153 key_len == OCS_AES_KEYSIZE_256))
154 return 0;
155
156 /* For SM4, only 128-byte keys are supported. */
157 if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128)
158 return 0;
159
160 /* Everything else is unsupported. */
161 return -EINVAL;
162}
163
164/* Save key into transformation context. */
165static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len,
166 enum ocs_cipher cipher)
167{
168 int ret;
169
170 ret = check_key(in_key, key_len, cipher);
171 if (ret)
172 return ret;
173
174 memcpy(tctx->key, in_key, key_len);
175 tctx->key_len = key_len;
176 tctx->cipher = cipher;
177
178 return 0;
179}
180
181/* Set key for symmetric cypher. */
182static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
183 size_t key_len, enum ocs_cipher cipher)
184{
185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
186
187 /* Fallback is used for AES with 192-bit key. */
188 tctx->use_fallback = (cipher == OCS_AES &&
189 key_len == OCS_AES_KEYSIZE_192);
190
191 if (!tctx->use_fallback)
192 return save_key(tctx, in_key, key_len, cipher);
193
194 crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk,
195 CRYPTO_TFM_REQ_MASK);
196 crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk,
197 tfm->base.crt_flags &
198 CRYPTO_TFM_REQ_MASK);
199
200 return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len);
201}
202
203/* Set key for AEAD cipher. */
204static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
205 size_t key_len, enum ocs_cipher cipher)
206{
207 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
208
209 /* Fallback is used for AES with 192-bit key. */
210 tctx->use_fallback = (cipher == OCS_AES &&
211 key_len == OCS_AES_KEYSIZE_192);
212
213 if (!tctx->use_fallback)
214 return save_key(tctx, in_key, key_len, cipher);
215
216 crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
217 crypto_aead_set_flags(tctx->sw_cipher.aead,
218 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
219
220 return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len);
221}
222
223/* Swap two AES blocks in SG lists. */
224static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents,
225 off_t blk1_offset, off_t blk2_offset)
226{
227 u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE];
228
229 /*
230 * No easy way to copy within sg list, so copy both blocks to temporary
231 * buffers first.
232 */
233 sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset);
234 sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset);
235 sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset);
236 sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset);
237}
238
239/* Initialize request context to default values. */
240static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx)
241{
242 /* Zero everything. */
243 memset(rctx, 0, sizeof(*rctx));
244
245 /* Set initial value for DMA addresses. */
246 rctx->src_dll.dma_addr = DMA_MAPPING_ERROR;
247 rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR;
248 rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR;
249 rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR;
250}
251
252static int kmb_ocs_sk_validate_input(struct skcipher_request *req,
253 enum ocs_mode mode)
254{
255 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
256 int iv_size = crypto_skcipher_ivsize(tfm);
257
258 switch (mode) {
259 case OCS_MODE_ECB:
260 /* Ensure input length is multiple of block size */
261 if (req->cryptlen % AES_BLOCK_SIZE != 0)
262 return -EINVAL;
263
264 return 0;
265
266 case OCS_MODE_CBC:
267 /* Ensure input length is multiple of block size */
268 if (req->cryptlen % AES_BLOCK_SIZE != 0)
269 return -EINVAL;
270
271 /* Ensure IV is present and block size in length */
272 if (!req->iv || iv_size != AES_BLOCK_SIZE)
273 return -EINVAL;
274 /*
275 * NOTE: Since req->cryptlen == 0 case was already handled in
276 * kmb_ocs_sk_common(), the above two conditions also guarantee
277 * that: cryptlen >= iv_size
278 */
279 return 0;
280
281 case OCS_MODE_CTR:
282 /* Ensure IV is present and block size in length */
283 if (!req->iv || iv_size != AES_BLOCK_SIZE)
284 return -EINVAL;
285 return 0;
286
287 case OCS_MODE_CTS:
288 /* Ensure input length >= block size */
289 if (req->cryptlen < AES_BLOCK_SIZE)
290 return -EINVAL;
291
292 /* Ensure IV is present and block size in length */
293 if (!req->iv || iv_size != AES_BLOCK_SIZE)
294 return -EINVAL;
295
296 return 0;
297 default:
298 return -EINVAL;
299 }
300}
301
302/*
303 * Called by encrypt() / decrypt() skcipher functions.
304 *
305 * Use fallback if needed, otherwise initialize context and enqueue request
306 * into engine.
307 */
308static int kmb_ocs_sk_common(struct skcipher_request *req,
309 enum ocs_cipher cipher,
310 enum ocs_instruction instruction,
311 enum ocs_mode mode)
312{
313 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
314 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
315 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
316 struct ocs_aes_dev *aes_dev;
317 int rc;
318
319 if (tctx->use_fallback) {
320 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk);
321
322 skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk);
323 skcipher_request_set_callback(subreq, req->base.flags, NULL,
324 NULL);
325 skcipher_request_set_crypt(subreq, req->src, req->dst,
326 req->cryptlen, req->iv);
327
328 if (instruction == OCS_ENCRYPT)
329 rc = crypto_skcipher_encrypt(subreq);
330 else
331 rc = crypto_skcipher_decrypt(subreq);
332
333 skcipher_request_zero(subreq);
334
335 return rc;
336 }
337
338 /*
339 * If cryptlen == 0, no processing needed for ECB, CBC and CTR.
340 *
341 * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL.
342 */
343 if (!req->cryptlen && mode != OCS_MODE_CTS)
344 return 0;
345
346 rc = kmb_ocs_sk_validate_input(req, mode);
347 if (rc)
348 return rc;
349
350 aes_dev = kmb_ocs_aes_find_dev(tctx);
351 if (!aes_dev)
352 return -ENODEV;
353
354 if (cipher != tctx->cipher)
355 return -EINVAL;
356
357 ocs_aes_init_rctx(rctx);
358 rctx->instruction = instruction;
359 rctx->mode = mode;
360
361 return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req);
362}
363
364static void cleanup_ocs_dma_linked_list(struct device *dev,
365 struct ocs_dll_desc *dll)
366{
367 if (dll->vaddr)
368 dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr);
369 dll->vaddr = NULL;
370 dll->size = 0;
371 dll->dma_addr = DMA_MAPPING_ERROR;
372}
373
374static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req)
375{
376 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
377 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
378 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
379 struct device *dev = tctx->aes_dev->dev;
380
381 if (rctx->src_dma_count) {
382 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
383 rctx->src_dma_count = 0;
384 }
385
386 if (rctx->dst_dma_count) {
387 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
388 DMA_BIDIRECTIONAL :
389 DMA_FROM_DEVICE);
390 rctx->dst_dma_count = 0;
391 }
392
393 /* Clean up OCS DMA linked lists */
394 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
395 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
396}
397
398static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req)
399{
400 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
401 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
402 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
403 int iv_size = crypto_skcipher_ivsize(tfm);
404 int rc;
405
406 /*
407 * For CBC decrypt, save last block (iv) to last_ct_blk buffer.
408 *
409 * Note: if we are here, we already checked that cryptlen >= iv_size
410 * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see
411 * kmb_ocs_sk_validate_input().
412 */
413 if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT)
414 scatterwalk_map_and_copy(rctx->last_ct_blk, req->src,
415 req->cryptlen - iv_size, iv_size, 0);
416
417 /* For CTS decrypt, swap last two blocks, if needed. */
418 if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT)
419 sg_swap_blocks(req->dst, rctx->dst_nents,
420 req->cryptlen - AES_BLOCK_SIZE,
421 req->cryptlen - (2 * AES_BLOCK_SIZE));
422
423 /* src and dst buffers are the same, use bidirectional DMA mapping. */
424 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
425 rctx->dst_nents, DMA_BIDIRECTIONAL);
426 if (rctx->dst_dma_count == 0) {
427 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
428 return -ENOMEM;
429 }
430
431 /* Create DST linked list */
432 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
433 rctx->dst_dma_count, &rctx->dst_dll,
434 req->cryptlen, 0);
435 if (rc)
436 return rc;
437 /*
438 * If descriptor creation was successful, set the src_dll.dma_addr to
439 * the value of dst_dll.dma_addr, as we do in-place AES operation on
440 * the src.
441 */
442 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
443
444 return 0;
445}
446
447static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req)
448{
449 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
450 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
451 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
452 int rc;
453
454 rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen);
455 if (rctx->src_nents < 0)
456 return -EBADMSG;
457
458 /* Map SRC SG. */
459 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
460 rctx->src_nents, DMA_TO_DEVICE);
461 if (rctx->src_dma_count == 0) {
462 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
463 return -ENOMEM;
464 }
465
466 /* Create SRC linked list */
467 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
468 rctx->src_dma_count, &rctx->src_dll,
469 req->cryptlen, 0);
470 if (rc)
471 return rc;
472
473 /* Map DST SG. */
474 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
475 rctx->dst_nents, DMA_FROM_DEVICE);
476 if (rctx->dst_dma_count == 0) {
477 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
478 return -ENOMEM;
479 }
480
481 /* Create DST linked list */
482 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
483 rctx->dst_dma_count, &rctx->dst_dll,
484 req->cryptlen, 0);
485 if (rc)
486 return rc;
487
488 /* If this is not a CTS decrypt operation with swapping, we are done. */
489 if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT))
490 return 0;
491
492 /*
493 * Otherwise, we have to copy src to dst (as we cannot modify src).
494 * Use OCS AES bypass mode to copy src to dst via DMA.
495 *
496 * NOTE: for anything other than small data sizes this is rather
497 * inefficient.
498 */
499 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr,
500 rctx->src_dll.dma_addr, req->cryptlen);
501 if (rc)
502 return rc;
503
504 /*
505 * Now dst == src, so clean up what we did so far and use in_place
506 * logic.
507 */
508 kmb_ocs_sk_dma_cleanup(req);
509 rctx->in_place = true;
510
511 return kmb_ocs_sk_prepare_inplace(req);
512}
513
514static int kmb_ocs_sk_run(struct skcipher_request *req)
515{
516 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
517 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
518 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
519 struct ocs_aes_dev *aes_dev = tctx->aes_dev;
520 int iv_size = crypto_skcipher_ivsize(tfm);
521 int rc;
522
523 rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
524 if (rctx->dst_nents < 0)
525 return -EBADMSG;
526
527 /*
528 * If 2 blocks or greater, and multiple of block size swap last two
529 * blocks to be compatible with other crypto API CTS implementations:
530 * OCS mode uses CBC-CS2, whereas other crypto API implementations use
531 * CBC-CS3.
532 * CBC-CS2 and CBC-CS3 defined by:
533 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf
534 */
535 rctx->cts_swap = (rctx->mode == OCS_MODE_CTS &&
536 req->cryptlen > AES_BLOCK_SIZE &&
537 req->cryptlen % AES_BLOCK_SIZE == 0);
538
539 rctx->in_place = (req->src == req->dst);
540
541 if (rctx->in_place)
542 rc = kmb_ocs_sk_prepare_inplace(req);
543 else
544 rc = kmb_ocs_sk_prepare_notinplace(req);
545
546 if (rc)
547 goto error;
548
549 rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction,
550 rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr,
551 req->cryptlen, req->iv, iv_size);
552 if (rc)
553 goto error;
554
555 /* Clean-up DMA before further processing output. */
556 kmb_ocs_sk_dma_cleanup(req);
557
558 /* For CTS Encrypt, swap last 2 blocks, if needed. */
559 if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) {
560 sg_swap_blocks(req->dst, rctx->dst_nents,
561 req->cryptlen - AES_BLOCK_SIZE,
562 req->cryptlen - (2 * AES_BLOCK_SIZE));
563 return 0;
564 }
565
566 /* For CBC copy IV to req->IV. */
567 if (rctx->mode == OCS_MODE_CBC) {
568 /* CBC encrypt case. */
569 if (rctx->instruction == OCS_ENCRYPT) {
570 scatterwalk_map_and_copy(req->iv, req->dst,
571 req->cryptlen - iv_size,
572 iv_size, 0);
573 return 0;
574 }
575 /* CBC decrypt case. */
576 if (rctx->in_place)
577 memcpy(req->iv, rctx->last_ct_blk, iv_size);
578 else
579 scatterwalk_map_and_copy(req->iv, req->src,
580 req->cryptlen - iv_size,
581 iv_size, 0);
582 return 0;
583 }
584 /* For all other modes there's nothing to do. */
585
586 return 0;
587
588error:
589 kmb_ocs_sk_dma_cleanup(req);
590
591 return rc;
592}
593
594static int kmb_ocs_aead_validate_input(struct aead_request *req,
595 enum ocs_instruction instruction,
596 enum ocs_mode mode)
597{
598 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
599 int tag_size = crypto_aead_authsize(tfm);
600 int iv_size = crypto_aead_ivsize(tfm);
601
602 /* For decrypt crytplen == len(PT) + len(tag). */
603 if (instruction == OCS_DECRYPT && req->cryptlen < tag_size)
604 return -EINVAL;
605
606 /* IV is mandatory. */
607 if (!req->iv)
608 return -EINVAL;
609
610 switch (mode) {
611 case OCS_MODE_GCM:
612 if (iv_size != GCM_AES_IV_SIZE)
613 return -EINVAL;
614
615 return 0;
616
617 case OCS_MODE_CCM:
618 /* Ensure IV is present and block size in length */
619 if (iv_size != AES_BLOCK_SIZE)
620 return -EINVAL;
621
622 return 0;
623
624 default:
625 return -EINVAL;
626 }
627}
628
629/*
630 * Called by encrypt() / decrypt() aead functions.
631 *
632 * Use fallback if needed, otherwise initialize context and enqueue request
633 * into engine.
634 */
635static int kmb_ocs_aead_common(struct aead_request *req,
636 enum ocs_cipher cipher,
637 enum ocs_instruction instruction,
638 enum ocs_mode mode)
639{
640 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
641 struct ocs_aes_rctx *rctx = aead_request_ctx(req);
642 struct ocs_aes_dev *dd;
643 int rc;
644
645 if (tctx->use_fallback) {
646 struct aead_request *subreq = aead_request_ctx(req);
647
648 aead_request_set_tfm(subreq, tctx->sw_cipher.aead);
649 aead_request_set_callback(subreq, req->base.flags,
650 req->base.complete, req->base.data);
651 aead_request_set_crypt(subreq, req->src, req->dst,
652 req->cryptlen, req->iv);
653 aead_request_set_ad(subreq, req->assoclen);
654 rc = crypto_aead_setauthsize(tctx->sw_cipher.aead,
655 crypto_aead_authsize(crypto_aead_reqtfm(req)));
656 if (rc)
657 return rc;
658
659 return (instruction == OCS_ENCRYPT) ?
660 crypto_aead_encrypt(subreq) :
661 crypto_aead_decrypt(subreq);
662 }
663
664 rc = kmb_ocs_aead_validate_input(req, instruction, mode);
665 if (rc)
666 return rc;
667
668 dd = kmb_ocs_aes_find_dev(tctx);
669 if (!dd)
670 return -ENODEV;
671
672 if (cipher != tctx->cipher)
673 return -EINVAL;
674
675 ocs_aes_init_rctx(rctx);
676 rctx->instruction = instruction;
677 rctx->mode = mode;
678
679 return crypto_transfer_aead_request_to_engine(dd->engine, req);
680}
681
682static void kmb_ocs_aead_dma_cleanup(struct aead_request *req)
683{
684 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
685 struct ocs_aes_rctx *rctx = aead_request_ctx(req);
686 struct device *dev = tctx->aes_dev->dev;
687
688 if (rctx->src_dma_count) {
689 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
690 rctx->src_dma_count = 0;
691 }
692
693 if (rctx->dst_dma_count) {
694 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
695 DMA_BIDIRECTIONAL :
696 DMA_FROM_DEVICE);
697 rctx->dst_dma_count = 0;
698 }
699 /* Clean up OCS DMA linked lists */
700 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
701 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
702 cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll);
703 cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll);
704}
705
706/**
707 * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing.
708 * @req: The AEAD request being processed.
709 * @src_dll_size: Where to store the length of the data mapped into the
710 * src_dll OCS DMA list.
711 *
712 * Do the following:
713 * - DMA map req->src and req->dst
714 * - Initialize the following OCS DMA linked lists: rctx->src_dll,
715 * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll.
716 *
717 * Return: 0 on success, negative error code otherwise.
718 */
719static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size)
720{
721 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
722 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
723 struct ocs_aes_rctx *rctx = aead_request_ctx(req);
724 u32 in_size; /* The length of the data to be mapped by src_dll. */
725 u32 out_size; /* The length of the data to be mapped by dst_dll. */
726 u32 dst_size; /* The length of the data in dst_sg. */
727 int rc;
728
729 /* Get number of entries in input data SG list. */
730 rctx->src_nents = sg_nents_for_len(req->src,
731 req->assoclen + req->cryptlen);
732 if (rctx->src_nents < 0)
733 return -EBADMSG;
734
735 if (rctx->instruction == OCS_DECRYPT) {
736 /*
737 * For decrypt:
738 * - src sg list is: AAD|CT|tag
739 * - dst sg list expects: AAD|PT
740 *
741 * in_size == len(CT); out_size == len(PT)
742 */
743
744 /* req->cryptlen includes both CT and tag. */
745 in_size = req->cryptlen - tag_size;
746
747 /* out_size = PT size == CT size */
748 out_size = in_size;
749
750 /* len(dst_sg) == len(AAD) + len(PT) */
751 dst_size = req->assoclen + out_size;
752
753 /*
754 * Copy tag from source SG list to 'in_tag' buffer.
755 *
756 * Note: this needs to be done here, before DMA mapping src_sg.
757 */
758 sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag,
759 tag_size, req->assoclen + in_size);
760
761 } else { /* OCS_ENCRYPT */
762 /*
763 * For encrypt:
764 * src sg list is: AAD|PT
765 * dst sg list expects: AAD|CT|tag
766 */
767 /* in_size == len(PT) */
768 in_size = req->cryptlen;
769
770 /*
771 * In CCM mode the OCS engine appends the tag to the ciphertext,
772 * but in GCM mode the tag must be read from the tag registers
773 * and appended manually below
774 */
775 out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size :
776 in_size;
777 /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */
778 dst_size = req->assoclen + in_size + tag_size;
779 }
780 *src_dll_size = in_size;
781
782 /* Get number of entries in output data SG list. */
783 rctx->dst_nents = sg_nents_for_len(req->dst, dst_size);
784 if (rctx->dst_nents < 0)
785 return -EBADMSG;
786
787 rctx->in_place = (req->src == req->dst) ? 1 : 0;
788
789 /* Map destination; use bidirectional mapping for in-place case. */
790 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
791 rctx->dst_nents,
792 rctx->in_place ? DMA_BIDIRECTIONAL :
793 DMA_FROM_DEVICE);
794 if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) {
795 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
796 return -ENOMEM;
797 }
798
799 /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */
800 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
801 rctx->dst_dma_count,
802 &rctx->aad_dst_dll, req->assoclen,
803 0);
804 if (rc)
805 return rc;
806
807 /* Create DST list: maps dst[AAD_SIZE:out_size] */
808 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
809 rctx->dst_dma_count, &rctx->dst_dll,
810 out_size, req->assoclen);
811 if (rc)
812 return rc;
813
814 if (rctx->in_place) {
815 /* If this is not CCM encrypt, we are done. */
816 if (!(rctx->mode == OCS_MODE_CCM &&
817 rctx->instruction == OCS_ENCRYPT)) {
818 /*
819 * SRC and DST are the same, so re-use the same DMA
820 * addresses (to avoid allocating new DMA lists
821 * identical to the dst ones).
822 */
823 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
824 rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr;
825
826 return 0;
827 }
828 /*
829 * For CCM encrypt the input and output linked lists contain
830 * different amounts of data, so, we need to create different
831 * SRC and AAD SRC lists, even for the in-place case.
832 */
833 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
834 rctx->dst_dma_count,
835 &rctx->aad_src_dll,
836 req->assoclen, 0);
837 if (rc)
838 return rc;
839 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
840 rctx->dst_dma_count,
841 &rctx->src_dll, in_size,
842 req->assoclen);
843 if (rc)
844 return rc;
845
846 return 0;
847 }
848 /* Not in-place case. */
849
850 /* Map source SG. */
851 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
852 rctx->src_nents, DMA_TO_DEVICE);
853 if (rctx->src_dma_count == 0 && rctx->src_nents != 0) {
854 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
855 return -ENOMEM;
856 }
857
858 /* Create AAD SRC list. */
859 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
860 rctx->src_dma_count,
861 &rctx->aad_src_dll,
862 req->assoclen, 0);
863 if (rc)
864 return rc;
865
866 /* Create SRC list. */
867 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
868 rctx->src_dma_count,
869 &rctx->src_dll, in_size,
870 req->assoclen);
871 if (rc)
872 return rc;
873
874 if (req->assoclen == 0)
875 return 0;
876
877 /* Copy AAD from src sg to dst sg using OCS DMA. */
878 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr,
879 rctx->aad_src_dll.dma_addr, req->cryptlen);
880 if (rc)
881 dev_err(tctx->aes_dev->dev,
882 "Failed to copy source AAD to destination AAD\n");
883
884 return rc;
885}
886
887static int kmb_ocs_aead_run(struct aead_request *req)
888{
889 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
890 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
891 struct ocs_aes_rctx *rctx = aead_request_ctx(req);
892 u32 in_size; /* The length of the data mapped by src_dll. */
893 int rc;
894
895 rc = kmb_ocs_aead_dma_prepare(req, &in_size);
896 if (rc)
897 goto exit;
898
899 /* For CCM, we just call the OCS processing and we are done. */
900 if (rctx->mode == OCS_MODE_CCM) {
901 rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher,
902 rctx->instruction, rctx->dst_dll.dma_addr,
903 rctx->src_dll.dma_addr, in_size,
904 req->iv,
905 rctx->aad_src_dll.dma_addr, req->assoclen,
906 rctx->in_tag, tag_size);
907 goto exit;
908 }
909 /* GCM case; invoke OCS processing. */
910 rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher,
911 rctx->instruction,
912 rctx->dst_dll.dma_addr,
913 rctx->src_dll.dma_addr, in_size,
914 req->iv,
915 rctx->aad_src_dll.dma_addr, req->assoclen,
916 rctx->out_tag, tag_size);
917 if (rc)
918 goto exit;
919
920 /* For GCM decrypt, we have to compare in_tag with out_tag. */
921 if (rctx->instruction == OCS_DECRYPT) {
922 rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ?
923 -EBADMSG : 0;
924 goto exit;
925 }
926
927 /* For GCM encrypt, we must manually copy out_tag to DST sg. */
928
929 /* Clean-up must be called before the sg_pcopy_from_buffer() below. */
930 kmb_ocs_aead_dma_cleanup(req);
931
932 /* Copy tag to destination sg after AAD and CT. */
933 sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag,
934 tag_size, req->assoclen + req->cryptlen);
935
936 /* Return directly as DMA cleanup already done. */
937 return 0;
938
939exit:
940 kmb_ocs_aead_dma_cleanup(req);
941
942 return rc;
943}
944
945static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine,
946 void *areq)
947{
948 struct skcipher_request *req =
949 container_of(areq, struct skcipher_request, base);
950 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
951 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
952 int err;
953
954 if (!tctx->aes_dev) {
955 err = -ENODEV;
956 goto exit;
957 }
958
959 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
960 tctx->cipher);
961 if (err)
962 goto exit;
963
964 err = kmb_ocs_sk_run(req);
965
966exit:
967 crypto_finalize_skcipher_request(engine, req, err);
968
969 return 0;
970}
971
972static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine,
973 void *areq)
974{
975 struct aead_request *req = container_of(areq,
976 struct aead_request, base);
977 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
978 int err;
979
980 if (!tctx->aes_dev)
981 return -ENODEV;
982
983 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
984 tctx->cipher);
985 if (err)
986 goto exit;
987
988 err = kmb_ocs_aead_run(req);
989
990exit:
991 crypto_finalize_aead_request(tctx->aes_dev->engine, req, err);
992
993 return 0;
994}
995
996static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
997 unsigned int key_len)
998{
999 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES);
1000}
1001
1002static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
1003 unsigned int key_len)
1004{
1005 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES);
1006}
1007
1008#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
1009static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req)
1010{
1011 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB);
1012}
1013
1014static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req)
1015{
1016 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB);
1017}
1018#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
1019
1020static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req)
1021{
1022 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC);
1023}
1024
1025static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req)
1026{
1027 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC);
1028}
1029
1030static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req)
1031{
1032 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR);
1033}
1034
1035static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req)
1036{
1037 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR);
1038}
1039
1040#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
1041static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req)
1042{
1043 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS);
1044}
1045
1046static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req)
1047{
1048 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS);
1049}
1050#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
1051
1052static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req)
1053{
1054 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM);
1055}
1056
1057static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req)
1058{
1059 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM);
1060}
1061
1062static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req)
1063{
1064 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM);
1065}
1066
1067static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req)
1068{
1069 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM);
1070}
1071
1072static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
1073 unsigned int key_len)
1074{
1075 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4);
1076}
1077
1078static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
1079 unsigned int key_len)
1080{
1081 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4);
1082}
1083
1084#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
1085static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req)
1086{
1087 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB);
1088}
1089
1090static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req)
1091{
1092 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB);
1093}
1094#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
1095
1096static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req)
1097{
1098 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC);
1099}
1100
1101static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req)
1102{
1103 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC);
1104}
1105
1106static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req)
1107{
1108 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR);
1109}
1110
1111static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req)
1112{
1113 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR);
1114}
1115
1116#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
1117static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req)
1118{
1119 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS);
1120}
1121
1122static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req)
1123{
1124 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS);
1125}
1126#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
1127
1128static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req)
1129{
1130 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM);
1131}
1132
1133static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req)
1134{
1135 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM);
1136}
1137
1138static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req)
1139{
1140 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM);
1141}
1142
1143static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req)
1144{
1145 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM);
1146}
1147
1148static int ocs_aes_init_tfm(struct crypto_skcipher *tfm)
1149{
1150 const char *alg_name = crypto_tfm_alg_name(&tfm->base);
1151 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
1152 struct crypto_sync_skcipher *blk;
1153
1154 /* set fallback cipher in case it will be needed */
1155 blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
1156 if (IS_ERR(blk))
1157 return PTR_ERR(blk);
1158
1159 tctx->sw_cipher.sk = blk;
1160
1161 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
1162
1163 return 0;
1164}
1165
1166static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm)
1167{
1168 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
1169
1170 return 0;
1171}
1172
1173static inline void clear_key(struct ocs_aes_tctx *tctx)
1174{
1175 memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256);
1176
1177 /* Zero key registers if set */
1178 if (tctx->aes_dev)
1179 ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256,
1180 tctx->key, OCS_AES);
1181}
1182
1183static void ocs_exit_tfm(struct crypto_skcipher *tfm)
1184{
1185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
1186
1187 clear_key(tctx);
1188
1189 if (tctx->sw_cipher.sk) {
1190 crypto_free_sync_skcipher(tctx->sw_cipher.sk);
1191 tctx->sw_cipher.sk = NULL;
1192 }
1193}
1194
1195static int ocs_aes_aead_cra_init(struct crypto_aead *tfm)
1196{
1197 const char *alg_name = crypto_tfm_alg_name(&tfm->base);
1198 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
1199 struct crypto_aead *blk;
1200
1201 /* Set fallback cipher in case it will be needed */
1202 blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
1203 if (IS_ERR(blk))
1204 return PTR_ERR(blk);
1205
1206 tctx->sw_cipher.aead = blk;
1207
1208 crypto_aead_set_reqsize(tfm,
1209 max(sizeof(struct ocs_aes_rctx),
1210 (sizeof(struct aead_request) +
1211 crypto_aead_reqsize(tctx->sw_cipher.aead))));
1212
1213 return 0;
1214}
1215
1216static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm,
1217 unsigned int authsize)
1218{
1219 switch (authsize) {
1220 case 4:
1221 case 6:
1222 case 8:
1223 case 10:
1224 case 12:
1225 case 14:
1226 case 16:
1227 return 0;
1228 default:
1229 return -EINVAL;
1230 }
1231}
1232
1233static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm,
1234 unsigned int authsize)
1235{
1236 return crypto_gcm_check_authsize(authsize);
1237}
1238
1239static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm)
1240{
1241 crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
1242
1243 return 0;
1244}
1245
1246static void ocs_aead_cra_exit(struct crypto_aead *tfm)
1247{
1248 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
1249
1250 clear_key(tctx);
1251
1252 if (tctx->sw_cipher.aead) {
1253 crypto_free_aead(tctx->sw_cipher.aead);
1254 tctx->sw_cipher.aead = NULL;
1255 }
1256}
1257
1258static struct skcipher_engine_alg algs[] = {
1259#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
1260 {
1261 .base.base.cra_name = "ecb(aes)",
1262 .base.base.cra_driver_name = "ecb-aes-keembay-ocs",
1263 .base.base.cra_priority = KMB_OCS_PRIORITY,
1264 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1265 CRYPTO_ALG_KERN_DRIVER_ONLY |
1266 CRYPTO_ALG_NEED_FALLBACK,
1267 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1268 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1269 .base.base.cra_module = THIS_MODULE,
1270 .base.base.cra_alignmask = 0,
1271
1272 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,
1273 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,
1274 .base.setkey = kmb_ocs_aes_set_key,
1275 .base.encrypt = kmb_ocs_aes_ecb_encrypt,
1276 .base.decrypt = kmb_ocs_aes_ecb_decrypt,
1277 .base.init = ocs_aes_init_tfm,
1278 .base.exit = ocs_exit_tfm,
1279 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1280 },
1281#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
1282 {
1283 .base.base.cra_name = "cbc(aes)",
1284 .base.base.cra_driver_name = "cbc-aes-keembay-ocs",
1285 .base.base.cra_priority = KMB_OCS_PRIORITY,
1286 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1287 CRYPTO_ALG_KERN_DRIVER_ONLY |
1288 CRYPTO_ALG_NEED_FALLBACK,
1289 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1290 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1291 .base.base.cra_module = THIS_MODULE,
1292 .base.base.cra_alignmask = 0,
1293
1294 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,
1295 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,
1296 .base.ivsize = AES_BLOCK_SIZE,
1297 .base.setkey = kmb_ocs_aes_set_key,
1298 .base.encrypt = kmb_ocs_aes_cbc_encrypt,
1299 .base.decrypt = kmb_ocs_aes_cbc_decrypt,
1300 .base.init = ocs_aes_init_tfm,
1301 .base.exit = ocs_exit_tfm,
1302 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1303 },
1304 {
1305 .base.base.cra_name = "ctr(aes)",
1306 .base.base.cra_driver_name = "ctr-aes-keembay-ocs",
1307 .base.base.cra_priority = KMB_OCS_PRIORITY,
1308 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1309 CRYPTO_ALG_KERN_DRIVER_ONLY |
1310 CRYPTO_ALG_NEED_FALLBACK,
1311 .base.base.cra_blocksize = 1,
1312 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1313 .base.base.cra_module = THIS_MODULE,
1314 .base.base.cra_alignmask = 0,
1315
1316 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,
1317 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,
1318 .base.ivsize = AES_BLOCK_SIZE,
1319 .base.setkey = kmb_ocs_aes_set_key,
1320 .base.encrypt = kmb_ocs_aes_ctr_encrypt,
1321 .base.decrypt = kmb_ocs_aes_ctr_decrypt,
1322 .base.init = ocs_aes_init_tfm,
1323 .base.exit = ocs_exit_tfm,
1324 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1325 },
1326#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
1327 {
1328 .base.base.cra_name = "cts(cbc(aes))",
1329 .base.base.cra_driver_name = "cts-aes-keembay-ocs",
1330 .base.base.cra_priority = KMB_OCS_PRIORITY,
1331 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1332 CRYPTO_ALG_KERN_DRIVER_ONLY |
1333 CRYPTO_ALG_NEED_FALLBACK,
1334 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1335 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1336 .base.base.cra_module = THIS_MODULE,
1337 .base.base.cra_alignmask = 0,
1338
1339 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,
1340 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,
1341 .base.ivsize = AES_BLOCK_SIZE,
1342 .base.setkey = kmb_ocs_aes_set_key,
1343 .base.encrypt = kmb_ocs_aes_cts_encrypt,
1344 .base.decrypt = kmb_ocs_aes_cts_decrypt,
1345 .base.init = ocs_aes_init_tfm,
1346 .base.exit = ocs_exit_tfm,
1347 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1348 },
1349#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
1350#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
1351 {
1352 .base.base.cra_name = "ecb(sm4)",
1353 .base.base.cra_driver_name = "ecb-sm4-keembay-ocs",
1354 .base.base.cra_priority = KMB_OCS_PRIORITY,
1355 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1356 CRYPTO_ALG_KERN_DRIVER_ONLY,
1357 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1358 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1359 .base.base.cra_module = THIS_MODULE,
1360 .base.base.cra_alignmask = 0,
1361
1362 .base.min_keysize = OCS_SM4_KEY_SIZE,
1363 .base.max_keysize = OCS_SM4_KEY_SIZE,
1364 .base.setkey = kmb_ocs_sm4_set_key,
1365 .base.encrypt = kmb_ocs_sm4_ecb_encrypt,
1366 .base.decrypt = kmb_ocs_sm4_ecb_decrypt,
1367 .base.init = ocs_sm4_init_tfm,
1368 .base.exit = ocs_exit_tfm,
1369 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1370 },
1371#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
1372 {
1373 .base.base.cra_name = "cbc(sm4)",
1374 .base.base.cra_driver_name = "cbc-sm4-keembay-ocs",
1375 .base.base.cra_priority = KMB_OCS_PRIORITY,
1376 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1377 CRYPTO_ALG_KERN_DRIVER_ONLY,
1378 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1379 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1380 .base.base.cra_module = THIS_MODULE,
1381 .base.base.cra_alignmask = 0,
1382
1383 .base.min_keysize = OCS_SM4_KEY_SIZE,
1384 .base.max_keysize = OCS_SM4_KEY_SIZE,
1385 .base.ivsize = AES_BLOCK_SIZE,
1386 .base.setkey = kmb_ocs_sm4_set_key,
1387 .base.encrypt = kmb_ocs_sm4_cbc_encrypt,
1388 .base.decrypt = kmb_ocs_sm4_cbc_decrypt,
1389 .base.init = ocs_sm4_init_tfm,
1390 .base.exit = ocs_exit_tfm,
1391 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1392 },
1393 {
1394 .base.base.cra_name = "ctr(sm4)",
1395 .base.base.cra_driver_name = "ctr-sm4-keembay-ocs",
1396 .base.base.cra_priority = KMB_OCS_PRIORITY,
1397 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1398 CRYPTO_ALG_KERN_DRIVER_ONLY,
1399 .base.base.cra_blocksize = 1,
1400 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1401 .base.base.cra_module = THIS_MODULE,
1402 .base.base.cra_alignmask = 0,
1403
1404 .base.min_keysize = OCS_SM4_KEY_SIZE,
1405 .base.max_keysize = OCS_SM4_KEY_SIZE,
1406 .base.ivsize = AES_BLOCK_SIZE,
1407 .base.setkey = kmb_ocs_sm4_set_key,
1408 .base.encrypt = kmb_ocs_sm4_ctr_encrypt,
1409 .base.decrypt = kmb_ocs_sm4_ctr_decrypt,
1410 .base.init = ocs_sm4_init_tfm,
1411 .base.exit = ocs_exit_tfm,
1412 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1413 },
1414#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
1415 {
1416 .base.base.cra_name = "cts(cbc(sm4))",
1417 .base.base.cra_driver_name = "cts-sm4-keembay-ocs",
1418 .base.base.cra_priority = KMB_OCS_PRIORITY,
1419 .base.base.cra_flags = CRYPTO_ALG_ASYNC |
1420 CRYPTO_ALG_KERN_DRIVER_ONLY,
1421 .base.base.cra_blocksize = AES_BLOCK_SIZE,
1422 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
1423 .base.base.cra_module = THIS_MODULE,
1424 .base.base.cra_alignmask = 0,
1425
1426 .base.min_keysize = OCS_SM4_KEY_SIZE,
1427 .base.max_keysize = OCS_SM4_KEY_SIZE,
1428 .base.ivsize = AES_BLOCK_SIZE,
1429 .base.setkey = kmb_ocs_sm4_set_key,
1430 .base.encrypt = kmb_ocs_sm4_cts_encrypt,
1431 .base.decrypt = kmb_ocs_sm4_cts_decrypt,
1432 .base.init = ocs_sm4_init_tfm,
1433 .base.exit = ocs_exit_tfm,
1434 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,
1435 }
1436#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
1437};
1438
1439static struct aead_engine_alg algs_aead[] = {
1440 {
1441 .base.base = {
1442 .cra_name = "gcm(aes)",
1443 .cra_driver_name = "gcm-aes-keembay-ocs",
1444 .cra_priority = KMB_OCS_PRIORITY,
1445 .cra_flags = CRYPTO_ALG_ASYNC |
1446 CRYPTO_ALG_KERN_DRIVER_ONLY |
1447 CRYPTO_ALG_NEED_FALLBACK,
1448 .cra_blocksize = 1,
1449 .cra_ctxsize = sizeof(struct ocs_aes_tctx),
1450 .cra_alignmask = 0,
1451 .cra_module = THIS_MODULE,
1452 },
1453 .base.init = ocs_aes_aead_cra_init,
1454 .base.exit = ocs_aead_cra_exit,
1455 .base.ivsize = GCM_AES_IV_SIZE,
1456 .base.maxauthsize = AES_BLOCK_SIZE,
1457 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize,
1458 .base.setkey = kmb_ocs_aes_aead_set_key,
1459 .base.encrypt = kmb_ocs_aes_gcm_encrypt,
1460 .base.decrypt = kmb_ocs_aes_gcm_decrypt,
1461 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,
1462 },
1463 {
1464 .base.base = {
1465 .cra_name = "ccm(aes)",
1466 .cra_driver_name = "ccm-aes-keembay-ocs",
1467 .cra_priority = KMB_OCS_PRIORITY,
1468 .cra_flags = CRYPTO_ALG_ASYNC |
1469 CRYPTO_ALG_KERN_DRIVER_ONLY |
1470 CRYPTO_ALG_NEED_FALLBACK,
1471 .cra_blocksize = 1,
1472 .cra_ctxsize = sizeof(struct ocs_aes_tctx),
1473 .cra_alignmask = 0,
1474 .cra_module = THIS_MODULE,
1475 },
1476 .base.init = ocs_aes_aead_cra_init,
1477 .base.exit = ocs_aead_cra_exit,
1478 .base.ivsize = AES_BLOCK_SIZE,
1479 .base.maxauthsize = AES_BLOCK_SIZE,
1480 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize,
1481 .base.setkey = kmb_ocs_aes_aead_set_key,
1482 .base.encrypt = kmb_ocs_aes_ccm_encrypt,
1483 .base.decrypt = kmb_ocs_aes_ccm_decrypt,
1484 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,
1485 },
1486 {
1487 .base.base = {
1488 .cra_name = "gcm(sm4)",
1489 .cra_driver_name = "gcm-sm4-keembay-ocs",
1490 .cra_priority = KMB_OCS_PRIORITY,
1491 .cra_flags = CRYPTO_ALG_ASYNC |
1492 CRYPTO_ALG_KERN_DRIVER_ONLY,
1493 .cra_blocksize = 1,
1494 .cra_ctxsize = sizeof(struct ocs_aes_tctx),
1495 .cra_alignmask = 0,
1496 .cra_module = THIS_MODULE,
1497 },
1498 .base.init = ocs_sm4_aead_cra_init,
1499 .base.exit = ocs_aead_cra_exit,
1500 .base.ivsize = GCM_AES_IV_SIZE,
1501 .base.maxauthsize = AES_BLOCK_SIZE,
1502 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize,
1503 .base.setkey = kmb_ocs_sm4_aead_set_key,
1504 .base.encrypt = kmb_ocs_sm4_gcm_encrypt,
1505 .base.decrypt = kmb_ocs_sm4_gcm_decrypt,
1506 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,
1507 },
1508 {
1509 .base.base = {
1510 .cra_name = "ccm(sm4)",
1511 .cra_driver_name = "ccm-sm4-keembay-ocs",
1512 .cra_priority = KMB_OCS_PRIORITY,
1513 .cra_flags = CRYPTO_ALG_ASYNC |
1514 CRYPTO_ALG_KERN_DRIVER_ONLY,
1515 .cra_blocksize = 1,
1516 .cra_ctxsize = sizeof(struct ocs_aes_tctx),
1517 .cra_alignmask = 0,
1518 .cra_module = THIS_MODULE,
1519 },
1520 .base.init = ocs_sm4_aead_cra_init,
1521 .base.exit = ocs_aead_cra_exit,
1522 .base.ivsize = AES_BLOCK_SIZE,
1523 .base.maxauthsize = AES_BLOCK_SIZE,
1524 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize,
1525 .base.setkey = kmb_ocs_sm4_aead_set_key,
1526 .base.encrypt = kmb_ocs_sm4_ccm_encrypt,
1527 .base.decrypt = kmb_ocs_sm4_ccm_decrypt,
1528 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,
1529 }
1530};
1531
1532static void unregister_aes_algs(struct ocs_aes_dev *aes_dev)
1533{
1534 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead));
1535 crypto_engine_unregister_skciphers(algs, ARRAY_SIZE(algs));
1536}
1537
1538static int register_aes_algs(struct ocs_aes_dev *aes_dev)
1539{
1540 int ret;
1541
1542 /*
1543 * If any algorithm fails to register, all preceding algorithms that
1544 * were successfully registered will be automatically unregistered.
1545 */
1546 ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead));
1547 if (ret)
1548 return ret;
1549
1550 ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs));
1551 if (ret)
1552 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs));
1553
1554 return ret;
1555}
1556
1557/* Device tree driver match. */
1558static const struct of_device_id kmb_ocs_aes_of_match[] = {
1559 {
1560 .compatible = "intel,keembay-ocs-aes",
1561 },
1562 {}
1563};
1564
1565static void kmb_ocs_aes_remove(struct platform_device *pdev)
1566{
1567 struct ocs_aes_dev *aes_dev;
1568
1569 aes_dev = platform_get_drvdata(pdev);
1570
1571 unregister_aes_algs(aes_dev);
1572
1573 spin_lock(&ocs_aes.lock);
1574 list_del(&aes_dev->list);
1575 spin_unlock(&ocs_aes.lock);
1576
1577 crypto_engine_exit(aes_dev->engine);
1578}
1579
1580static int kmb_ocs_aes_probe(struct platform_device *pdev)
1581{
1582 struct device *dev = &pdev->dev;
1583 struct ocs_aes_dev *aes_dev;
1584 int rc;
1585
1586 aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL);
1587 if (!aes_dev)
1588 return -ENOMEM;
1589
1590 aes_dev->dev = dev;
1591
1592 platform_set_drvdata(pdev, aes_dev);
1593
1594 rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1595 if (rc) {
1596 dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc);
1597 return rc;
1598 }
1599
1600 /* Get base register address. */
1601 aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);
1602 if (IS_ERR(aes_dev->base_reg))
1603 return PTR_ERR(aes_dev->base_reg);
1604
1605 /* Get and request IRQ */
1606 aes_dev->irq = platform_get_irq(pdev, 0);
1607 if (aes_dev->irq < 0)
1608 return aes_dev->irq;
1609
1610 rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler,
1611 NULL, 0, "keembay-ocs-aes", aes_dev);
1612 if (rc < 0) {
1613 dev_err(dev, "Could not request IRQ\n");
1614 return rc;
1615 }
1616
1617 INIT_LIST_HEAD(&aes_dev->list);
1618 spin_lock(&ocs_aes.lock);
1619 list_add_tail(&aes_dev->list, &ocs_aes.dev_list);
1620 spin_unlock(&ocs_aes.lock);
1621
1622 init_completion(&aes_dev->irq_completion);
1623
1624 /* Initialize crypto engine */
1625 aes_dev->engine = crypto_engine_alloc_init(dev, true);
1626 if (!aes_dev->engine) {
1627 rc = -ENOMEM;
1628 goto list_del;
1629 }
1630
1631 rc = crypto_engine_start(aes_dev->engine);
1632 if (rc) {
1633 dev_err(dev, "Could not start crypto engine\n");
1634 goto cleanup;
1635 }
1636
1637 rc = register_aes_algs(aes_dev);
1638 if (rc) {
1639 dev_err(dev,
1640 "Could not register OCS algorithms with Crypto API\n");
1641 goto cleanup;
1642 }
1643
1644 return 0;
1645
1646cleanup:
1647 crypto_engine_exit(aes_dev->engine);
1648list_del:
1649 spin_lock(&ocs_aes.lock);
1650 list_del(&aes_dev->list);
1651 spin_unlock(&ocs_aes.lock);
1652
1653 return rc;
1654}
1655
1656/* The OCS driver is a platform device. */
1657static struct platform_driver kmb_ocs_aes_driver = {
1658 .probe = kmb_ocs_aes_probe,
1659 .remove_new = kmb_ocs_aes_remove,
1660 .driver = {
1661 .name = DRV_NAME,
1662 .of_match_table = kmb_ocs_aes_of_match,
1663 },
1664};
1665
1666module_platform_driver(kmb_ocs_aes_driver);
1667
1668MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver");
1669MODULE_LICENSE("GPL");
1670
1671MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs");
1672MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs");
1673MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs");
1674MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs");
1675
1676MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs");
1677MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs");
1678MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs");
1679MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs");
1680
1681#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
1682MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs");
1683MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs");
1684#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
1685
1686#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
1687MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs");
1688MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs");
1689#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */