1// SPDX-License-Identifier: GPL-2.0-or-later
  2/**
  3 * AMCC SoC PPC4xx Crypto Driver
  4 *
  5 * Copyright (c) 2008 Applied Micro Circuits Corporation.
  6 * All rights reserved. James Hsiao <jhsiao@amcc.com>
  7 *
  8 * This file implements the Linux crypto algorithms.
  9 */
 10
 11#include <linux/kernel.h>
 12#include <linux/interrupt.h>
 13#include <linux/spinlock_types.h>
 14#include <linux/scatterlist.h>
 15#include <linux/crypto.h>
 16#include <linux/hash.h>
 17#include <crypto/internal/hash.h>
 18#include <linux/dma-mapping.h>
 19#include <crypto/algapi.h>
 20#include <crypto/aead.h>
 21#include <crypto/aes.h>
 22#include <crypto/gcm.h>
 23#include <crypto/sha.h>
 24#include <crypto/ctr.h>
 25#include <crypto/skcipher.h>
 26#include "crypto4xx_reg_def.h"
 27#include "crypto4xx_core.h"
 28#include "crypto4xx_sa.h"
 29
 30static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
 31				     u32 save_iv, u32 ld_h, u32 ld_iv,
 32				     u32 hdr_proc, u32 h, u32 c, u32 pad_type,
 33				     u32 op_grp, u32 op, u32 dir)
 34{
 35	sa->sa_command_0.w = 0;
 36	sa->sa_command_0.bf.save_hash_state = save_h;
 37	sa->sa_command_0.bf.save_iv = save_iv;
 38	sa->sa_command_0.bf.load_hash_state = ld_h;
 39	sa->sa_command_0.bf.load_iv = ld_iv;
 40	sa->sa_command_0.bf.hdr_proc = hdr_proc;
 41	sa->sa_command_0.bf.hash_alg = h;
 42	sa->sa_command_0.bf.cipher_alg = c;
 43	sa->sa_command_0.bf.pad_type = pad_type & 3;
 44	sa->sa_command_0.bf.extend_pad = pad_type >> 2;
 45	sa->sa_command_0.bf.op_group = op_grp;
 46	sa->sa_command_0.bf.opcode = op;
 47	sa->sa_command_0.bf.dir = dir;
 48}
 49
 50static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
 51				     u32 hmac_mc, u32 cfb, u32 esn,
 52				     u32 sn_mask, u32 mute, u32 cp_pad,
 53				     u32 cp_pay, u32 cp_hdr)
 54{
 55	sa->sa_command_1.w = 0;
 56	sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
 57	sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
 58	sa->sa_command_1.bf.feedback_mode = cfb,
 59	sa->sa_command_1.bf.sa_rev = 1;
 60	sa->sa_command_1.bf.hmac_muting = hmac_mc;
 61	sa->sa_command_1.bf.extended_seq_num = esn;
 62	sa->sa_command_1.bf.seq_num_mask = sn_mask;
 63	sa->sa_command_1.bf.mutable_bit_proc = mute;
 64	sa->sa_command_1.bf.copy_pad = cp_pad;
 65	sa->sa_command_1.bf.copy_payload = cp_pay;
 66	sa->sa_command_1.bf.copy_hdr = cp_hdr;
 67}
 68
 69static inline int crypto4xx_crypt(struct skcipher_request *req,
 70				  const unsigned int ivlen, bool decrypt,
 71				  bool check_blocksize)
 72{
 73	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 74	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
 75	__le32 iv[AES_IV_SIZE];
 76
 77	if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
 78		return -EINVAL;
 79
 80	if (ivlen)
 81		crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
 82
 83	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
 84		req->cryptlen, iv, ivlen, decrypt ? ctx->sa_in : ctx->sa_out,
 85		ctx->sa_len, 0, NULL);
 86}
 87
 88int crypto4xx_encrypt_noiv_block(struct skcipher_request *req)
 89{
 90	return crypto4xx_crypt(req, 0, false, true);
 91}
 92
 93int crypto4xx_encrypt_iv_stream(struct skcipher_request *req)
 94{
 95	return crypto4xx_crypt(req, AES_IV_SIZE, false, false);
 96}
 97
 98int crypto4xx_decrypt_noiv_block(struct skcipher_request *req)
 99{
100	return crypto4xx_crypt(req, 0, true, true);
101}
102
103int crypto4xx_decrypt_iv_stream(struct skcipher_request *req)
104{
105	return crypto4xx_crypt(req, AES_IV_SIZE, true, false);
106}
107
108int crypto4xx_encrypt_iv_block(struct skcipher_request *req)
109{
110	return crypto4xx_crypt(req, AES_IV_SIZE, false, true);
111}
112
113int crypto4xx_decrypt_iv_block(struct skcipher_request *req)
114{
115	return crypto4xx_crypt(req, AES_IV_SIZE, true, true);
116}
117
118/**
119 * AES Functions
120 */
121static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher,
122				const u8 *key,
123				unsigned int keylen,
124				unsigned char cm,
125				u8 fb)
126{
127	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
128	struct dynamic_sa_ctl *sa;
129	int    rc;
130
131	if (keylen != AES_KEYSIZE_256 &&
132		keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_128) {
133		crypto_skcipher_set_flags(cipher,
134				CRYPTO_TFM_RES_BAD_KEY_LEN);
135		return -EINVAL;
136	}
137
138	/* Create SA */
139	if (ctx->sa_in || ctx->sa_out)
140		crypto4xx_free_sa(ctx);
141
142	rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4);
143	if (rc)
144		return rc;
145
146	/* Setup SA */
147	sa = ctx->sa_in;
148
149	set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_ECB ?
150				 SA_NOT_SAVE_IV : SA_SAVE_IV),
151				 SA_NOT_LOAD_HASH, (cm == CRYPTO_MODE_ECB ?
152				 SA_LOAD_IV_FROM_SA : SA_LOAD_IV_FROM_STATE),
153				 SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
154				 SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
155				 SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT,
156				 DIR_INBOUND);
157
158	set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH,
159				 fb, SA_EXTENDED_SN_OFF,
160				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
161				 SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
162				 SA_NOT_COPY_HDR);
163	crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa),
164				 key, keylen);
165	sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2);
166	sa->sa_command_1.bf.key_len = keylen >> 3;
167
168	memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
169	sa = ctx->sa_out;
170	sa->sa_command_0.bf.dir = DIR_OUTBOUND;
171	/*
172	 * SA_OPCODE_ENCRYPT is the same value as SA_OPCODE_DECRYPT.
173	 * it's the DIR_(IN|OUT)BOUND that matters
174	 */
175	sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT;
176
177	return 0;
178}
179
180int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher,
181			     const u8 *key, unsigned int keylen)
182{
183	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
184				    CRYPTO_FEEDBACK_MODE_NO_FB);
185}
186
187int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher,
188			     const u8 *key, unsigned int keylen)
189{
190	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB,
191				    CRYPTO_FEEDBACK_MODE_128BIT_CFB);
192}
193
194int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher,
195			     const u8 *key, unsigned int keylen)
196{
197	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB,
198				    CRYPTO_FEEDBACK_MODE_NO_FB);
199}
200
201int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher,
202			     const u8 *key, unsigned int keylen)
203{
204	return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB,
205				    CRYPTO_FEEDBACK_MODE_64BIT_OFB);
206}
207
208int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
209			     const u8 *key, unsigned int keylen)
210{
211	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
212	int rc;
213
214	rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE,
215		CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB);
216	if (rc)
217		return rc;
218
219	ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen -
220						 CTR_RFC3686_NONCE_SIZE]);
221
222	return 0;
223}
224
225int crypto4xx_rfc3686_encrypt(struct skcipher_request *req)
226{
227	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
228	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
229	__le32 iv[AES_IV_SIZE / 4] = {
230		ctx->iv_nonce,
231		cpu_to_le32p((u32 *) req->iv),
232		cpu_to_le32p((u32 *) (req->iv + 4)),
233		cpu_to_le32(1) };
234
235	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
236				  req->cryptlen, iv, AES_IV_SIZE,
237				  ctx->sa_out, ctx->sa_len, 0, NULL);
238}
239
240int crypto4xx_rfc3686_decrypt(struct skcipher_request *req)
241{
242	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
243	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
244	__le32 iv[AES_IV_SIZE / 4] = {
245		ctx->iv_nonce,
246		cpu_to_le32p((u32 *) req->iv),
247		cpu_to_le32p((u32 *) (req->iv + 4)),
248		cpu_to_le32(1) };
249
250	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
251				  req->cryptlen, iv, AES_IV_SIZE,
252				  ctx->sa_out, ctx->sa_len, 0, NULL);
253}
254
255static int
256crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
257{
258	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
259	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
260	size_t iv_len = crypto_skcipher_ivsize(cipher);
261	unsigned int counter = be32_to_cpup((__be32 *)(req->iv + iv_len - 4));
262	unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
263			AES_BLOCK_SIZE;
264
265	/*
266	 * The hardware uses only the last 32-bits as the counter while the
267	 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
268	 * the whole IV is a counter.  So fallback if the counter is going to
269	 * overlow.
270	 */
271	if (counter + nblks < counter) {
272		SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher.cipher);
273		int ret;
274
275		skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher.cipher);
276		skcipher_request_set_callback(subreq, req->base.flags,
277			NULL, NULL);
278		skcipher_request_set_crypt(subreq, req->src, req->dst,
279			req->cryptlen, req->iv);
280		ret = encrypt ? crypto_skcipher_encrypt(subreq)
281			: crypto_skcipher_decrypt(subreq);
282		skcipher_request_zero(subreq);
283		return ret;
284	}
285
286	return encrypt ? crypto4xx_encrypt_iv_stream(req)
287		       : crypto4xx_decrypt_iv_stream(req);
288}
289
290static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,
291				       struct crypto_skcipher *cipher,
292				       const u8 *key,
293				       unsigned int keylen)
294{
295	int rc;
296
297	crypto_sync_skcipher_clear_flags(ctx->sw_cipher.cipher,
298				    CRYPTO_TFM_REQ_MASK);
299	crypto_sync_skcipher_set_flags(ctx->sw_cipher.cipher,
300		crypto_skcipher_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
301	rc = crypto_sync_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen);
302	crypto_skcipher_clear_flags(cipher, CRYPTO_TFM_RES_MASK);
303	crypto_skcipher_set_flags(cipher,
304		crypto_sync_skcipher_get_flags(ctx->sw_cipher.cipher) &
305			CRYPTO_TFM_RES_MASK);
306
307	return rc;
308}
309
310int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher,
311			     const u8 *key, unsigned int keylen)
312{
313	struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
314	int rc;
315
316	rc = crypto4xx_sk_setup_fallback(ctx, cipher, key, keylen);
317	if (rc)
318		return rc;
319
320	return crypto4xx_setkey_aes(cipher, key, keylen,
321		CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB);
322}
323
324int crypto4xx_encrypt_ctr(struct skcipher_request *req)
325{
326	return crypto4xx_ctr_crypt(req, true);
327}
328
329int crypto4xx_decrypt_ctr(struct skcipher_request *req)
330{
331	return crypto4xx_ctr_crypt(req, false);
332}
333
334static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
335						unsigned int len,
336						bool is_ccm, bool decrypt)
337{
338	struct crypto_aead *aead = crypto_aead_reqtfm(req);
339
340	/* authsize has to be a multiple of 4 */
341	if (aead->authsize & 3)
342		return true;
343
344	/*
345	 * hardware does not handle cases where plaintext
346	 * is less than a block.
347	 */
348	if (len < AES_BLOCK_SIZE)
349		return true;
350
351	/* assoc len needs to be a multiple of 4 and <= 1020 */
352	if (req->assoclen & 0x3 || req->assoclen > 1020)
353		return true;
354
355	/* CCM supports only counter field length of 2 and 4 bytes */
356	if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3))
357		return true;
358
359	return false;
360}
361
362static int crypto4xx_aead_fallback(struct aead_request *req,
363	struct crypto4xx_ctx *ctx, bool do_decrypt)
364{
365	struct aead_request *subreq = aead_request_ctx(req);
366
367	aead_request_set_tfm(subreq, ctx->sw_cipher.aead);
368	aead_request_set_callback(subreq, req->base.flags,
369				  req->base.complete, req->base.data);
370	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
371			       req->iv);
372	aead_request_set_ad(subreq, req->assoclen);
373	return do_decrypt ? crypto_aead_decrypt(subreq) :
374			    crypto_aead_encrypt(subreq);
375}
376
377static int crypto4xx_aead_setup_fallback(struct crypto4xx_ctx *ctx,
378					 struct crypto_aead *cipher,
379					 const u8 *key,
380					 unsigned int keylen)
381{
382	int rc;
383
384	crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
385	crypto_aead_set_flags(ctx->sw_cipher.aead,
386		crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK);
387	rc = crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen);
388	crypto_aead_clear_flags(cipher, CRYPTO_TFM_RES_MASK);
389	crypto_aead_set_flags(cipher,
390		crypto_aead_get_flags(ctx->sw_cipher.aead) &
391			CRYPTO_TFM_RES_MASK);
392
393	return rc;
394}
395
396/**
397 * AES-CCM Functions
398 */
399
400int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key,
401			     unsigned int keylen)
402{
403	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
404	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
405	struct dynamic_sa_ctl *sa;
406	int rc = 0;
407
408	rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
409	if (rc)
410		return rc;
411
412	if (ctx->sa_in || ctx->sa_out)
413		crypto4xx_free_sa(ctx);
414
415	rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4);
416	if (rc)
417		return rc;
418
419	/* Setup SA */
420	sa = (struct dynamic_sa_ctl *) ctx->sa_in;
421	sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2);
422
423	set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
424				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
425				 SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC,
426				 SA_CIPHER_ALG_AES,
427				 SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
428				 SA_OPCODE_HASH_DECRYPT, DIR_INBOUND);
429
430	set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
431				 CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
432				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
433				 SA_NOT_COPY_PAD, SA_COPY_PAYLOAD,
434				 SA_NOT_COPY_HDR);
435
436	sa->sa_command_1.bf.key_len = keylen >> 3;
437
438	crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen);
439
440	memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
441	sa = (struct dynamic_sa_ctl *) ctx->sa_out;
442
443	set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
444				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
445				 SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC,
446				 SA_CIPHER_ALG_AES,
447				 SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
448				 SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND);
449
450	set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
451				 CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
452				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
453				 SA_COPY_PAD, SA_COPY_PAYLOAD,
454				 SA_NOT_COPY_HDR);
455
456	sa->sa_command_1.bf.key_len = keylen >> 3;
457	return 0;
458}
459
460static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt)
461{
462	struct crypto4xx_ctx *ctx  = crypto_tfm_ctx(req->base.tfm);
463	struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
464	struct crypto_aead *aead = crypto_aead_reqtfm(req);
465	__le32 iv[16];
466	u32 tmp_sa[SA_AES128_CCM_LEN + 4];
467	struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa;
468	unsigned int len = req->cryptlen;
469
470	if (decrypt)
471		len -= crypto_aead_authsize(aead);
472
473	if (crypto4xx_aead_need_fallback(req, len, true, decrypt))
474		return crypto4xx_aead_fallback(req, ctx, decrypt);
475
476	memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, ctx->sa_len * 4);
477	sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2;
478
479	if (req->iv[0] == 1) {
480		/* CRYPTO_MODE_AES_ICM */
481		sa->sa_command_1.bf.crypto_mode9_8 = 1;
482	}
483
484	iv[3] = cpu_to_le32(0);
485	crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1));
486
487	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
488				  len, iv, sizeof(iv),
489				  sa, ctx->sa_len, req->assoclen, rctx->dst);
490}
491
492int crypto4xx_encrypt_aes_ccm(struct aead_request *req)
493{
494	return crypto4xx_crypt_aes_ccm(req, false);
495}
496
497int crypto4xx_decrypt_aes_ccm(struct aead_request *req)
498{
499	return crypto4xx_crypt_aes_ccm(req, true);
500}
501
502int crypto4xx_setauthsize_aead(struct crypto_aead *cipher,
503			       unsigned int authsize)
504{
505	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
506	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
507
508	return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize);
509}
510
511/**
512 * AES-GCM Functions
513 */
514
515static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen)
516{
517	switch (keylen) {
518	case 16:
519	case 24:
520	case 32:
521		return 0;
522	default:
523		return -EINVAL;
524	}
525}
526
527static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key,
528					     unsigned int keylen)
529{
530	struct crypto_aes_ctx ctx;
531	uint8_t src[16] = { 0 };
532	int rc;
533
534	rc = aes_expandkey(&ctx, key, keylen);
535	if (rc) {
536		pr_err("aes_expandkey() failed: %d\n", rc);
537		return rc;
538	}
539
540	aes_encrypt(&ctx, src, src);
541	crypto4xx_memcpy_to_le32(hash_start, src, 16);
542	memzero_explicit(&ctx, sizeof(ctx));
543	return 0;
544}
545
546int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher,
547			     const u8 *key, unsigned int keylen)
548{
549	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
550	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
551	struct dynamic_sa_ctl *sa;
552	int    rc = 0;
553
554	if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0) {
555		crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
556		return -EINVAL;
557	}
558
559	rc = crypto4xx_aead_setup_fallback(ctx, cipher, key, keylen);
560	if (rc)
561		return rc;
562
563	if (ctx->sa_in || ctx->sa_out)
564		crypto4xx_free_sa(ctx);
565
566	rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4);
567	if (rc)
568		return rc;
569
570	sa  = (struct dynamic_sa_ctl *) ctx->sa_in;
571
572	sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2);
573	set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
574				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
575				 SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH,
576				 SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
577				 SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT,
578				 DIR_INBOUND);
579	set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH,
580				 CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
581				 SA_SEQ_MASK_ON, SA_MC_DISABLE,
582				 SA_NOT_COPY_PAD, SA_COPY_PAYLOAD,
583				 SA_NOT_COPY_HDR);
584
585	sa->sa_command_1.bf.key_len = keylen >> 3;
586
587	crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa),
588				 key, keylen);
589
590	rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa),
591		key, keylen);
592	if (rc) {
593		pr_err("GCM hash key setting failed = %d\n", rc);
594		goto err;
595	}
596
597	memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
598	sa = (struct dynamic_sa_ctl *) ctx->sa_out;
599	sa->sa_command_0.bf.dir = DIR_OUTBOUND;
600	sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH;
601
602	return 0;
603err:
604	crypto4xx_free_sa(ctx);
605	return rc;
606}
607
608static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req,
609					  bool decrypt)
610{
611	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
612	struct crypto4xx_aead_reqctx *rctx = aead_request_ctx(req);
613	__le32 iv[4];
614	unsigned int len = req->cryptlen;
615
616	if (decrypt)
617		len -= crypto_aead_authsize(crypto_aead_reqtfm(req));
618
619	if (crypto4xx_aead_need_fallback(req, len, false, decrypt))
620		return crypto4xx_aead_fallback(req, ctx, decrypt);
621
622	crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE);
623	iv[3] = cpu_to_le32(1);
624
625	return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
626				  len, iv, sizeof(iv),
627				  decrypt ? ctx->sa_in : ctx->sa_out,
628				  ctx->sa_len, req->assoclen, rctx->dst);
629}
630
631int crypto4xx_encrypt_aes_gcm(struct aead_request *req)
632{
633	return crypto4xx_crypt_aes_gcm(req, false);
634}
635
636int crypto4xx_decrypt_aes_gcm(struct aead_request *req)
637{
638	return crypto4xx_crypt_aes_gcm(req, true);
639}
640
641/**
642 * HASH SHA1 Functions
643 */
644static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm,
645				   unsigned int sa_len,
646				   unsigned char ha,
647				   unsigned char hm)
648{
649	struct crypto_alg *alg = tfm->__crt_alg;
650	struct crypto4xx_alg *my_alg;
651	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
652	struct dynamic_sa_hash160 *sa;
653	int rc;
654
655	my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg,
656			      alg.u.hash);
657	ctx->dev   = my_alg->dev;
658
659	/* Create SA */
660	if (ctx->sa_in || ctx->sa_out)
661		crypto4xx_free_sa(ctx);
662
663	rc = crypto4xx_alloc_sa(ctx, sa_len);
664	if (rc)
665		return rc;
666
667	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
668				 sizeof(struct crypto4xx_ctx));
669	sa = (struct dynamic_sa_hash160 *)ctx->sa_in;
670	set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV,
671				 SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA,
672				 SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL,
673				 SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
674				 SA_OPCODE_HASH, DIR_INBOUND);
675	set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH,
676				 CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
677				 SA_SEQ_MASK_OFF, SA_MC_ENABLE,
678				 SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
679				 SA_NOT_COPY_HDR);
680	/* Need to zero hash digest in SA */
681	memset(sa->inner_digest, 0, sizeof(sa->inner_digest));
682	memset(sa->outer_digest, 0, sizeof(sa->outer_digest));
683
684	return 0;
685}
686
687int crypto4xx_hash_init(struct ahash_request *req)
688{
689	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
690	int ds;
691	struct dynamic_sa_ctl *sa;
692
693	sa = ctx->sa_in;
694	ds = crypto_ahash_digestsize(
695			__crypto_ahash_cast(req->base.tfm));
696	sa->sa_command_0.bf.digest_len = ds >> 2;
697	sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA;
698
699	return 0;
700}
701
702int crypto4xx_hash_update(struct ahash_request *req)
703{
704	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
705	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
706	struct scatterlist dst;
707	unsigned int ds = crypto_ahash_digestsize(ahash);
708
709	sg_init_one(&dst, req->result, ds);
710
711	return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
712				  req->nbytes, NULL, 0, ctx->sa_in,
713				  ctx->sa_len, 0, NULL);
714}
715
716int crypto4xx_hash_final(struct ahash_request *req)
717{
718	return 0;
719}
720
721int crypto4xx_hash_digest(struct ahash_request *req)
722{
723	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
724	struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
725	struct scatterlist dst;
726	unsigned int ds = crypto_ahash_digestsize(ahash);
727
728	sg_init_one(&dst, req->result, ds);
729
730	return crypto4xx_build_pd(&req->base, ctx, req->src, &dst,
731				  req->nbytes, NULL, 0, ctx->sa_in,
732				  ctx->sa_len, 0, NULL);
733}
734
735/**
736 * SHA1 Algorithm
737 */
738int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm)
739{
740	return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1,
741				       SA_HASH_MODE_HASH);
742}