Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Cryptographic API.
4 *
5 * Support for OMAP AES GCM HW acceleration.
6 *
7 * Copyright (c) 2016 Texas Instruments Incorporated
8 */
9
10#include <linux/errno.h>
11#include <linux/scatterlist.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h>
14#include <linux/omap-dma.h>
15#include <linux/interrupt.h>
16#include <linux/pm_runtime.h>
17#include <crypto/aes.h>
18#include <crypto/gcm.h>
19#include <crypto/scatterwalk.h>
20#include <crypto/skcipher.h>
21#include <crypto/internal/aead.h>
22
23#include "omap-crypto.h"
24#include "omap-aes.h"
25
26static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
27 struct aead_request *req);
28
29static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
30{
31 struct aead_request *req = dd->aead_req;
32
33 dd->in_sg = NULL;
34 dd->out_sg = NULL;
35
36 crypto_finalize_aead_request(dd->engine, req, ret);
37
38 pm_runtime_mark_last_busy(dd->dev);
39 pm_runtime_put_autosuspend(dd->dev);
40}
41
42static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
43{
44 u8 *tag;
45 int alen, clen, i, ret = 0, nsg;
46 struct omap_aes_reqctx *rctx;
47
48 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
49 clen = ALIGN(dd->total, AES_BLOCK_SIZE);
50 rctx = aead_request_ctx(dd->aead_req);
51
52 nsg = !!(dd->assoc_len && dd->total);
53
54 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
55 DMA_FROM_DEVICE);
56 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
57 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
58 omap_aes_crypt_dma_stop(dd);
59
60 omap_crypto_cleanup(dd->out_sg, dd->orig_out,
61 dd->aead_req->assoclen, dd->total,
62 FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
63
64 if (dd->flags & FLAGS_ENCRYPT)
65 scatterwalk_map_and_copy(rctx->auth_tag,
66 dd->aead_req->dst,
67 dd->total + dd->aead_req->assoclen,
68 dd->authsize, 1);
69
70 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
71 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
72
73 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
74 FLAGS_IN_DATA_ST_SHIFT, dd->flags);
75
76 if (!(dd->flags & FLAGS_ENCRYPT)) {
77 tag = (u8 *)rctx->auth_tag;
78 for (i = 0; i < dd->authsize; i++) {
79 if (tag[i]) {
80 ret = -EBADMSG;
81 }
82 }
83 }
84
85 omap_aes_gcm_finish_req(dd, ret);
86}
87
88static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
89 struct aead_request *req)
90{
91 int alen, clen, cryptlen, assoclen, ret;
92 struct crypto_aead *aead = crypto_aead_reqtfm(req);
93 unsigned int authlen = crypto_aead_authsize(aead);
94 struct scatterlist *tmp, sg_arr[2];
95 int nsg;
96 u16 flags;
97
98 assoclen = req->assoclen;
99 cryptlen = req->cryptlen;
100
101 if (dd->flags & FLAGS_RFC4106_GCM)
102 assoclen -= 8;
103
104 if (!(dd->flags & FLAGS_ENCRYPT))
105 cryptlen -= authlen;
106
107 alen = ALIGN(assoclen, AES_BLOCK_SIZE);
108 clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
109
110 nsg = !!(assoclen && cryptlen);
111
112 omap_aes_clear_copy_flags(dd);
113
114 sg_init_table(dd->in_sgl, nsg + 1);
115 if (assoclen) {
116 tmp = req->src;
117 ret = omap_crypto_align_sg(&tmp, assoclen,
118 AES_BLOCK_SIZE, dd->in_sgl,
119 OMAP_CRYPTO_COPY_DATA |
120 OMAP_CRYPTO_ZERO_BUF |
121 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
122 FLAGS_ASSOC_DATA_ST_SHIFT,
123 &dd->flags);
124 if (ret)
125 return ret;
126 }
127
128 if (cryptlen) {
129 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
130
131 if (nsg)
132 sg_unmark_end(dd->in_sgl);
133
134 ret = omap_crypto_align_sg(&tmp, cryptlen,
135 AES_BLOCK_SIZE, &dd->in_sgl[nsg],
136 OMAP_CRYPTO_COPY_DATA |
137 OMAP_CRYPTO_ZERO_BUF |
138 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
139 FLAGS_IN_DATA_ST_SHIFT,
140 &dd->flags);
141 if (ret)
142 return ret;
143 }
144
145 dd->in_sg = dd->in_sgl;
146 dd->total = cryptlen;
147 dd->assoc_len = assoclen;
148 dd->authsize = authlen;
149
150 dd->out_sg = req->dst;
151 dd->orig_out = req->dst;
152
153 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
154
155 flags = 0;
156 if (req->src == req->dst || dd->out_sg == sg_arr)
157 flags |= OMAP_CRYPTO_FORCE_COPY;
158
159 if (cryptlen) {
160 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
161 AES_BLOCK_SIZE, &dd->out_sgl,
162 flags,
163 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
164 if (ret)
165 return ret;
166 }
167
168 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
169 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
170
171 return 0;
172}
173
174static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
175{
176 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
177
178 aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
179 return 0;
180}
181
182void omap_aes_gcm_dma_out_callback(void *data)
183{
184 struct omap_aes_dev *dd = data;
185 struct omap_aes_reqctx *rctx;
186 int i, val;
187 u32 *auth_tag, tag[4];
188
189 if (!(dd->flags & FLAGS_ENCRYPT))
190 scatterwalk_map_and_copy(tag, dd->aead_req->src,
191 dd->total + dd->aead_req->assoclen,
192 dd->authsize, 0);
193
194 rctx = aead_request_ctx(dd->aead_req);
195 auth_tag = (u32 *)rctx->auth_tag;
196 for (i = 0; i < 4; i++) {
197 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
198 auth_tag[i] = val ^ auth_tag[i];
199 if (!(dd->flags & FLAGS_ENCRYPT))
200 auth_tag[i] = auth_tag[i] ^ tag[i];
201 }
202
203 omap_aes_gcm_done_task(dd);
204}
205
206static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
207 struct aead_request *req)
208{
209 if (req)
210 return crypto_transfer_aead_request_to_engine(dd->engine, req);
211
212 return 0;
213}
214
215static int omap_aes_gcm_prepare_req(struct crypto_engine *engine, void *areq)
216{
217 struct aead_request *req = container_of(areq, struct aead_request,
218 base);
219 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
220 struct omap_aes_dev *dd = rctx->dd;
221 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
222 int err;
223
224 dd->aead_req = req;
225
226 rctx->mode &= FLAGS_MODE_MASK;
227 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
228
229 err = omap_aes_gcm_copy_buffers(dd, req);
230 if (err)
231 return err;
232
233 dd->ctx = &ctx->octx;
234
235 return omap_aes_write_ctrl(dd);
236}
237
238static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
239{
240 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
241 struct crypto_aead *aead = crypto_aead_reqtfm(req);
242 unsigned int authlen = crypto_aead_authsize(aead);
243 struct omap_aes_dev *dd;
244 __be32 counter = cpu_to_be32(1);
245 int err, assoclen;
246
247 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
248 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
249
250 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
251 if (err)
252 return err;
253
254 if (mode & FLAGS_RFC4106_GCM)
255 assoclen = req->assoclen - 8;
256 else
257 assoclen = req->assoclen;
258 if (assoclen + req->cryptlen == 0) {
259 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
260 1);
261 return 0;
262 }
263
264 dd = omap_aes_find_dev(rctx);
265 if (!dd)
266 return -ENODEV;
267 rctx->mode = mode;
268
269 return omap_aes_gcm_handle_queue(dd, req);
270}
271
272int omap_aes_gcm_encrypt(struct aead_request *req)
273{
274 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
275
276 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
277 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
278}
279
280int omap_aes_gcm_decrypt(struct aead_request *req)
281{
282 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
283
284 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
285 return omap_aes_gcm_crypt(req, FLAGS_GCM);
286}
287
288int omap_aes_4106gcm_encrypt(struct aead_request *req)
289{
290 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
291 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
292
293 memcpy(rctx->iv, ctx->octx.nonce, 4);
294 memcpy(rctx->iv + 4, req->iv, 8);
295 return crypto_ipsec_check_assoclen(req->assoclen) ?:
296 omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
297 FLAGS_RFC4106_GCM);
298}
299
300int omap_aes_4106gcm_decrypt(struct aead_request *req)
301{
302 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
303 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
304
305 memcpy(rctx->iv, ctx->octx.nonce, 4);
306 memcpy(rctx->iv + 4, req->iv, 8);
307 return crypto_ipsec_check_assoclen(req->assoclen) ?:
308 omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
309}
310
311int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
312 unsigned int keylen)
313{
314 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
315 int ret;
316
317 ret = aes_expandkey(&ctx->actx, key, keylen);
318 if (ret)
319 return ret;
320
321 memcpy(ctx->octx.key, key, keylen);
322 ctx->octx.keylen = keylen;
323
324 return 0;
325}
326
327int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
328 unsigned int keylen)
329{
330 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
331 int ret;
332
333 if (keylen < 4)
334 return -EINVAL;
335 keylen -= 4;
336
337 ret = aes_expandkey(&ctx->actx, key, keylen);
338 if (ret)
339 return ret;
340
341 memcpy(ctx->octx.key, key, keylen);
342 memcpy(ctx->octx.nonce, key + keylen, 4);
343 ctx->octx.keylen = keylen;
344
345 return 0;
346}
347
348int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
349{
350 return crypto_gcm_check_authsize(authsize);
351}
352
353int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
354 unsigned int authsize)
355{
356 return crypto_rfc4106_check_authsize(authsize);
357}
358
359static int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
360{
361 struct aead_request *req = container_of(areq, struct aead_request,
362 base);
363 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
364 struct omap_aes_dev *dd = rctx->dd;
365 int ret = 0;
366
367 if (!dd)
368 return -ENODEV;
369
370 if (dd->in_sg_len)
371 ret = omap_aes_crypt_dma_start(dd);
372 else
373 omap_aes_gcm_dma_out_callback(dd);
374
375 return ret;
376}
377
378int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
379{
380 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
381
382 ctx->enginectx.op.prepare_request = omap_aes_gcm_prepare_req;
383 ctx->enginectx.op.unprepare_request = NULL;
384 ctx->enginectx.op.do_one_request = omap_aes_gcm_crypt_req;
385
386 crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
387
388 return 0;
389}