Linux Audio

Check our new training course

Embedded Linux training

Mar 10-20, 2025, special US time zones
Register
Loading...
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2016-2017 HiSilicon Limited. */
   3#include <linux/crypto.h>
   4#include <linux/dma-mapping.h>
   5#include <linux/dmapool.h>
   6#include <linux/module.h>
   7#include <linux/mutex.h>
   8#include <linux/slab.h>
   9
  10#include <crypto/aes.h>
  11#include <crypto/algapi.h>
  12#include <crypto/internal/des.h>
  13#include <crypto/skcipher.h>
  14#include <crypto/xts.h>
  15#include <crypto/internal/skcipher.h>
  16
  17#include "sec_drv.h"
  18
  19#define SEC_MAX_CIPHER_KEY		64
  20#define SEC_REQ_LIMIT SZ_32M
  21
  22struct sec_c_alg_cfg {
  23	unsigned c_alg		: 3;
  24	unsigned c_mode		: 3;
  25	unsigned key_len	: 2;
  26	unsigned c_width	: 2;
  27};
  28
  29static const struct sec_c_alg_cfg sec_c_alg_cfgs[] =  {
  30	[SEC_C_DES_ECB_64] = {
  31		.c_alg = SEC_C_ALG_DES,
  32		.c_mode = SEC_C_MODE_ECB,
  33		.key_len = SEC_KEY_LEN_DES,
  34	},
  35	[SEC_C_DES_CBC_64] = {
  36		.c_alg = SEC_C_ALG_DES,
  37		.c_mode = SEC_C_MODE_CBC,
  38		.key_len = SEC_KEY_LEN_DES,
  39	},
  40	[SEC_C_3DES_ECB_192_3KEY] = {
  41		.c_alg = SEC_C_ALG_3DES,
  42		.c_mode = SEC_C_MODE_ECB,
  43		.key_len = SEC_KEY_LEN_3DES_3_KEY,
  44	},
  45	[SEC_C_3DES_ECB_192_2KEY] = {
  46		.c_alg = SEC_C_ALG_3DES,
  47		.c_mode = SEC_C_MODE_ECB,
  48		.key_len = SEC_KEY_LEN_3DES_2_KEY,
  49	},
  50	[SEC_C_3DES_CBC_192_3KEY] = {
  51		.c_alg = SEC_C_ALG_3DES,
  52		.c_mode = SEC_C_MODE_CBC,
  53		.key_len = SEC_KEY_LEN_3DES_3_KEY,
  54	},
  55	[SEC_C_3DES_CBC_192_2KEY] = {
  56		.c_alg = SEC_C_ALG_3DES,
  57		.c_mode = SEC_C_MODE_CBC,
  58		.key_len = SEC_KEY_LEN_3DES_2_KEY,
  59	},
  60	[SEC_C_AES_ECB_128] = {
  61		.c_alg = SEC_C_ALG_AES,
  62		.c_mode = SEC_C_MODE_ECB,
  63		.key_len = SEC_KEY_LEN_AES_128,
  64	},
  65	[SEC_C_AES_ECB_192] = {
  66		.c_alg = SEC_C_ALG_AES,
  67		.c_mode = SEC_C_MODE_ECB,
  68		.key_len = SEC_KEY_LEN_AES_192,
  69	},
  70	[SEC_C_AES_ECB_256] = {
  71		.c_alg = SEC_C_ALG_AES,
  72		.c_mode = SEC_C_MODE_ECB,
  73		.key_len = SEC_KEY_LEN_AES_256,
  74	},
  75	[SEC_C_AES_CBC_128] = {
  76		.c_alg = SEC_C_ALG_AES,
  77		.c_mode = SEC_C_MODE_CBC,
  78		.key_len = SEC_KEY_LEN_AES_128,
  79	},
  80	[SEC_C_AES_CBC_192] = {
  81		.c_alg = SEC_C_ALG_AES,
  82		.c_mode = SEC_C_MODE_CBC,
  83		.key_len = SEC_KEY_LEN_AES_192,
  84	},
  85	[SEC_C_AES_CBC_256] = {
  86		.c_alg = SEC_C_ALG_AES,
  87		.c_mode = SEC_C_MODE_CBC,
  88		.key_len = SEC_KEY_LEN_AES_256,
  89	},
  90	[SEC_C_AES_CTR_128] = {
  91		.c_alg = SEC_C_ALG_AES,
  92		.c_mode = SEC_C_MODE_CTR,
  93		.key_len = SEC_KEY_LEN_AES_128,
  94	},
  95	[SEC_C_AES_CTR_192] = {
  96		.c_alg = SEC_C_ALG_AES,
  97		.c_mode = SEC_C_MODE_CTR,
  98		.key_len = SEC_KEY_LEN_AES_192,
  99	},
 100	[SEC_C_AES_CTR_256] = {
 101		.c_alg = SEC_C_ALG_AES,
 102		.c_mode = SEC_C_MODE_CTR,
 103		.key_len = SEC_KEY_LEN_AES_256,
 104	},
 105	[SEC_C_AES_XTS_128] = {
 106		.c_alg = SEC_C_ALG_AES,
 107		.c_mode = SEC_C_MODE_XTS,
 108		.key_len = SEC_KEY_LEN_AES_128,
 109	},
 110	[SEC_C_AES_XTS_256] = {
 111		.c_alg = SEC_C_ALG_AES,
 112		.c_mode = SEC_C_MODE_XTS,
 113		.key_len = SEC_KEY_LEN_AES_256,
 114	},
 115	[SEC_C_NULL] = {
 116	},
 117};
 118
 119/*
 120 * Mutex used to ensure safe operation of reference count of
 121 * alg providers
 122 */
 123static DEFINE_MUTEX(algs_lock);
 124static unsigned int active_devs;
 125
 126static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx,
 127					   struct sec_bd_info *req,
 128					   enum sec_cipher_alg alg)
 129{
 130	const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
 131
 132	memset(req, 0, sizeof(*req));
 133	req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S;
 134	req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S;
 135	req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S;
 136	req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S;
 137
 138	req->cipher_key_addr_lo = lower_32_bits(ctx->pkey);
 139	req->cipher_key_addr_hi = upper_32_bits(ctx->pkey);
 140}
 141
 142static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
 143					  const u8 *key,
 144					  unsigned int keylen,
 145					  enum sec_cipher_alg alg)
 146{
 147	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
 148	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 149
 150	ctx->cipher_alg = alg;
 151	memcpy(ctx->key, key, keylen);
 152	sec_alg_skcipher_init_template(ctx, &ctx->req_template,
 153				       ctx->cipher_alg);
 154}
 155
 156static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl,
 157			    dma_addr_t psec_sgl, struct sec_dev_info *info)
 158{
 159	struct sec_hw_sgl *sgl_current, *sgl_next;
 160	dma_addr_t sgl_next_dma;
 161
 162	sgl_current = hw_sgl;
 163	while (sgl_current) {
 164		sgl_next = sgl_current->next;
 165		sgl_next_dma = sgl_current->next_sgl;
 166
 167		dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl);
 168
 169		sgl_current = sgl_next;
 170		psec_sgl = sgl_next_dma;
 171	}
 172}
 173
 174static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl,
 175				     dma_addr_t *psec_sgl,
 176				     struct scatterlist *sgl,
 177				     int count,
 178				     struct sec_dev_info *info,
 179				     gfp_t gfp)
 180{
 181	struct sec_hw_sgl *sgl_current = NULL;
 182	struct sec_hw_sgl *sgl_next;
 183	dma_addr_t sgl_next_dma;
 184	struct scatterlist *sg;
 185	int ret, sge_index, i;
 186
 187	if (!count)
 188		return -EINVAL;
 189
 190	for_each_sg(sgl, sg, count, i) {
 191		sge_index = i % SEC_MAX_SGE_NUM;
 192		if (sge_index == 0) {
 193			sgl_next = dma_pool_zalloc(info->hw_sgl_pool,
 194						   gfp, &sgl_next_dma);
 195			if (!sgl_next) {
 196				ret = -ENOMEM;
 197				goto err_free_hw_sgls;
 198			}
 199
 200			if (!sgl_current) { /* First one */
 201				*psec_sgl = sgl_next_dma;
 202				*sec_sgl = sgl_next;
 203			} else { /* Chained */
 204				sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
 205				sgl_current->next_sgl = sgl_next_dma;
 206				sgl_current->next = sgl_next;
 207			}
 208			sgl_current = sgl_next;
 209		}
 210		sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg);
 211		sgl_current->sge_entries[sge_index].len = sg_dma_len(sg);
 212		sgl_current->data_bytes_in_sgl += sg_dma_len(sg);
 213	}
 214	sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
 215	sgl_current->next_sgl = 0;
 216	(*sec_sgl)->entry_sum_in_chain = count;
 217
 218	return 0;
 219
 220err_free_hw_sgls:
 221	sec_free_hw_sgl(*sec_sgl, *psec_sgl, info);
 222	*psec_sgl = 0;
 223
 224	return ret;
 225}
 226
 227static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
 228				   const u8 *key, unsigned int keylen,
 229				   enum sec_cipher_alg alg)
 230{
 231	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 232	struct device *dev = ctx->queue->dev_info->dev;
 233
 234	mutex_lock(&ctx->lock);
 235	if (ctx->key) {
 236		/* rekeying */
 237		memset(ctx->key, 0, SEC_MAX_CIPHER_KEY);
 238	} else {
 239		/* new key */
 240		ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY,
 241					      &ctx->pkey, GFP_KERNEL);
 242		if (!ctx->key) {
 243			mutex_unlock(&ctx->lock);
 244			return -ENOMEM;
 245		}
 246	}
 247	mutex_unlock(&ctx->lock);
 248	sec_alg_skcipher_init_context(tfm, key, keylen, alg);
 249
 250	return 0;
 251}
 252
 253static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
 254					   const u8 *key, unsigned int keylen)
 255{
 256	enum sec_cipher_alg alg;
 257
 258	switch (keylen) {
 259	case AES_KEYSIZE_128:
 260		alg = SEC_C_AES_ECB_128;
 261		break;
 262	case AES_KEYSIZE_192:
 263		alg = SEC_C_AES_ECB_192;
 264		break;
 265	case AES_KEYSIZE_256:
 266		alg = SEC_C_AES_ECB_256;
 267		break;
 268	default:
 269		return -EINVAL;
 270	}
 271
 272	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 273}
 274
 275static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
 276					   const u8 *key, unsigned int keylen)
 277{
 278	enum sec_cipher_alg alg;
 279
 280	switch (keylen) {
 281	case AES_KEYSIZE_128:
 282		alg = SEC_C_AES_CBC_128;
 283		break;
 284	case AES_KEYSIZE_192:
 285		alg = SEC_C_AES_CBC_192;
 286		break;
 287	case AES_KEYSIZE_256:
 288		alg = SEC_C_AES_CBC_256;
 289		break;
 290	default:
 291		return -EINVAL;
 292	}
 293
 294	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 295}
 296
 297static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
 298					   const u8 *key, unsigned int keylen)
 299{
 300	enum sec_cipher_alg alg;
 301
 302	switch (keylen) {
 303	case AES_KEYSIZE_128:
 304		alg = SEC_C_AES_CTR_128;
 305		break;
 306	case AES_KEYSIZE_192:
 307		alg = SEC_C_AES_CTR_192;
 308		break;
 309	case AES_KEYSIZE_256:
 310		alg = SEC_C_AES_CTR_256;
 311		break;
 312	default:
 313		return -EINVAL;
 314	}
 315
 316	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 317}
 318
 319static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
 320					   const u8 *key, unsigned int keylen)
 321{
 322	enum sec_cipher_alg alg;
 323	int ret;
 324
 325	ret = xts_verify_key(tfm, key, keylen);
 326	if (ret)
 327		return ret;
 328
 329	switch (keylen) {
 330	case AES_KEYSIZE_128 * 2:
 331		alg = SEC_C_AES_XTS_128;
 332		break;
 333	case AES_KEYSIZE_256 * 2:
 334		alg = SEC_C_AES_XTS_256;
 335		break;
 336	default:
 337		return -EINVAL;
 338	}
 339
 340	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 341}
 342
 343static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
 344					   const u8 *key, unsigned int keylen)
 345{
 346	return verify_skcipher_des_key(tfm, key) ?:
 347	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
 348}
 349
 350static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
 351					   const u8 *key, unsigned int keylen)
 352{
 353	return verify_skcipher_des_key(tfm, key) ?:
 354	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
 355}
 356
 357static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
 358					    const u8 *key, unsigned int keylen)
 359{
 360	return verify_skcipher_des3_key(tfm, key) ?:
 361	       sec_alg_skcipher_setkey(tfm, key, keylen,
 362				       SEC_C_3DES_ECB_192_3KEY);
 363}
 364
 365static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
 366					    const u8 *key, unsigned int keylen)
 367{
 368	return verify_skcipher_des3_key(tfm, key) ?:
 369	       sec_alg_skcipher_setkey(tfm, key, keylen,
 370				       SEC_C_3DES_CBC_192_3KEY);
 371}
 372
 373static void sec_alg_free_el(struct sec_request_el *el,
 374			    struct sec_dev_info *info)
 375{
 376	sec_free_hw_sgl(el->out, el->dma_out, info);
 377	sec_free_hw_sgl(el->in, el->dma_in, info);
 378	kfree(el->sgl_in);
 379	kfree(el->sgl_out);
 380	kfree(el);
 381}
 382
 383/* queuelock must be held */
 384static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue)
 385{
 386	struct sec_request_el *el, *temp;
 387	int ret = 0;
 388
 389	mutex_lock(&sec_req->lock);
 390	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
 391		/*
 392		 * Add to hardware queue only under following circumstances
 393		 * 1) Software and hardware queue empty so no chain dependencies
 394		 * 2) No dependencies as new IV - (check software queue empty
 395		 *    to maintain order)
 396		 * 3) No dependencies because the mode does no chaining.
 397		 *
 398		 * In other cases first insert onto the software queue which
 399		 * is then emptied as requests complete
 400		 */
 401		if (!queue->havesoftqueue ||
 402		    (kfifo_is_empty(&queue->softqueue) &&
 403		     sec_queue_empty(queue))) {
 404			ret = sec_queue_send(queue, &el->req, sec_req);
 405			if (ret == -EAGAIN) {
 406				/* Wait unti we can send then try again */
 407				/* DEAD if here - should not happen */
 408				ret = -EBUSY;
 409				goto err_unlock;
 410			}
 411		} else {
 412			kfifo_put(&queue->softqueue, el);
 413		}
 414	}
 415err_unlock:
 416	mutex_unlock(&sec_req->lock);
 417
 418	return ret;
 419}
 420
 421static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
 422				      struct crypto_async_request *req_base)
 423{
 424	struct skcipher_request *skreq = container_of(req_base,
 425						      struct skcipher_request,
 426						      base);
 427	struct sec_request *sec_req = skcipher_request_ctx(skreq);
 428	struct sec_request *backlog_req;
 429	struct sec_request_el *sec_req_el, *nextrequest;
 430	struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx;
 431	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
 432	struct device *dev = ctx->queue->dev_info->dev;
 433	int icv_or_skey_en, ret;
 434	bool done;
 435
 436	sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el,
 437				      head);
 438	icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
 439		SEC_BD_W0_ICV_OR_SKEY_EN_S;
 440	if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
 441		dev_err(dev, "Got an invalid answer %lu %d\n",
 442			sec_resp->w1 & SEC_BD_W1_BD_INVALID,
 443			icv_or_skey_en);
 444		sec_req->err = -EINVAL;
 445		/*
 446		 * We need to muddle on to avoid getting stuck with elements
 447		 * on the queue. Error will be reported so requester so
 448		 * it should be able to handle appropriately.
 449		 */
 450	}
 451
 452	spin_lock_bh(&ctx->queue->queuelock);
 453	/* Put the IV in place for chained cases */
 454	switch (ctx->cipher_alg) {
 455	case SEC_C_AES_CBC_128:
 456	case SEC_C_AES_CBC_192:
 457	case SEC_C_AES_CBC_256:
 458		if (sec_req_el->req.w0 & SEC_BD_W0_DE)
 459			sg_pcopy_to_buffer(sec_req_el->sgl_out,
 460					   sg_nents(sec_req_el->sgl_out),
 461					   skreq->iv,
 462					   crypto_skcipher_ivsize(atfm),
 463					   sec_req_el->el_length -
 464					   crypto_skcipher_ivsize(atfm));
 465		else
 466			sg_pcopy_to_buffer(sec_req_el->sgl_in,
 467					   sg_nents(sec_req_el->sgl_in),
 468					   skreq->iv,
 469					   crypto_skcipher_ivsize(atfm),
 470					   sec_req_el->el_length -
 471					   crypto_skcipher_ivsize(atfm));
 472		/* No need to sync to the device as coherent DMA */
 473		break;
 474	case SEC_C_AES_CTR_128:
 475	case SEC_C_AES_CTR_192:
 476	case SEC_C_AES_CTR_256:
 477		crypto_inc(skreq->iv, 16);
 478		break;
 479	default:
 480		/* Do not update */
 481		break;
 482	}
 483
 484	if (ctx->queue->havesoftqueue &&
 485	    !kfifo_is_empty(&ctx->queue->softqueue) &&
 486	    sec_queue_empty(ctx->queue)) {
 487		ret = kfifo_get(&ctx->queue->softqueue, &nextrequest);
 488		if (ret <= 0)
 489			dev_err(dev,
 490				"Error getting next element from kfifo %d\n",
 491				ret);
 492		else
 493			/* We know there is space so this cannot fail */
 494			sec_queue_send(ctx->queue, &nextrequest->req,
 495				       nextrequest->sec_req);
 496	} else if (!list_empty(&ctx->backlog)) {
 497		/* Need to verify there is room first */
 498		backlog_req = list_first_entry(&ctx->backlog,
 499					       typeof(*backlog_req),
 500					       backlog_head);
 501		if (sec_queue_can_enqueue(ctx->queue,
 502		    backlog_req->num_elements) ||
 503		    (ctx->queue->havesoftqueue &&
 504		     kfifo_avail(&ctx->queue->softqueue) >
 505		     backlog_req->num_elements)) {
 506			sec_send_request(backlog_req, ctx->queue);
 507			crypto_request_complete(backlog_req->req_base,
 508						-EINPROGRESS);
 509			list_del(&backlog_req->backlog_head);
 510		}
 511	}
 512	spin_unlock_bh(&ctx->queue->queuelock);
 513
 514	mutex_lock(&sec_req->lock);
 515	list_del(&sec_req_el->head);
 516	mutex_unlock(&sec_req->lock);
 517	sec_alg_free_el(sec_req_el, ctx->queue->dev_info);
 518
 519	/*
 520	 * Request is done.
 521	 * The dance is needed as the lock is freed in the completion
 522	 */
 523	mutex_lock(&sec_req->lock);
 524	done = list_empty(&sec_req->elements);
 525	mutex_unlock(&sec_req->lock);
 526	if (done) {
 527		if (crypto_skcipher_ivsize(atfm)) {
 528			dma_unmap_single(dev, sec_req->dma_iv,
 529					 crypto_skcipher_ivsize(atfm),
 530					 DMA_TO_DEVICE);
 531		}
 532		dma_unmap_sg(dev, skreq->src, sec_req->len_in,
 533			     DMA_BIDIRECTIONAL);
 534		if (skreq->src != skreq->dst)
 535			dma_unmap_sg(dev, skreq->dst, sec_req->len_out,
 536				     DMA_BIDIRECTIONAL);
 537		skcipher_request_complete(skreq, sec_req->err);
 538	}
 539}
 540
 541void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
 542{
 543	struct sec_request *sec_req = shadow;
 544
 545	sec_req->cb(resp, sec_req->req_base);
 546}
 547
 548static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes,
 549					      int *steps, gfp_t gfp)
 550{
 551	size_t *sizes;
 552	int i;
 553
 554	/* Split into suitable sized blocks */
 555	*steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT;
 556	sizes = kcalloc(*steps, sizeof(*sizes), gfp);
 557	if (!sizes)
 558		return -ENOMEM;
 559
 560	for (i = 0; i < *steps - 1; i++)
 561		sizes[i] = SEC_REQ_LIMIT;
 562	sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1);
 563	*split_sizes = sizes;
 564
 565	return 0;
 566}
 567
 568static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes,
 569				int steps, struct scatterlist ***splits,
 570				int **splits_nents,
 571				int sgl_len_in,
 572				struct device *dev, gfp_t gfp)
 573{
 574	int ret, count;
 575
 576	count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 577	if (!count)
 578		return -EINVAL;
 579
 580	*splits = kcalloc(steps, sizeof(struct scatterlist *), gfp);
 581	if (!*splits) {
 582		ret = -ENOMEM;
 583		goto err_unmap_sg;
 584	}
 585	*splits_nents = kcalloc(steps, sizeof(int), gfp);
 586	if (!*splits_nents) {
 587		ret = -ENOMEM;
 588		goto err_free_splits;
 589	}
 590
 591	/* output the scatter list before and after this */
 592	ret = sg_split(sgl, count, 0, steps, split_sizes,
 593		       *splits, *splits_nents, gfp);
 594	if (ret) {
 595		ret = -ENOMEM;
 596		goto err_free_splits_nents;
 597	}
 598
 599	return 0;
 600
 601err_free_splits_nents:
 602	kfree(*splits_nents);
 603err_free_splits:
 604	kfree(*splits);
 605err_unmap_sg:
 606	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 607
 608	return ret;
 609}
 610
 611/*
 612 * Reverses the sec_map_and_split_sg call for messages not yet added to
 613 * the queues.
 614 */
 615static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps,
 616				struct scatterlist **splits, int *splits_nents,
 617				int sgl_len_in, struct device *dev)
 618{
 619	int i;
 620
 621	for (i = 0; i < steps; i++)
 622		kfree(splits[i]);
 623	kfree(splits_nents);
 624	kfree(splits);
 625
 626	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 627}
 628
 629static struct sec_request_el
 630*sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt,
 631			   int el_size, bool different_dest,
 632			   struct scatterlist *sgl_in, int n_ents_in,
 633			   struct scatterlist *sgl_out, int n_ents_out,
 634			   struct sec_dev_info *info, gfp_t gfp)
 635{
 636	struct sec_request_el *el;
 637	struct sec_bd_info *req;
 638	int ret;
 639
 640	el = kzalloc(sizeof(*el), gfp);
 641	if (!el)
 642		return ERR_PTR(-ENOMEM);
 643	el->el_length = el_size;
 644	req = &el->req;
 645	memcpy(req, template, sizeof(*req));
 646
 647	req->w0 &= ~SEC_BD_W0_CIPHER_M;
 648	if (encrypt)
 649		req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S;
 650	else
 651		req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S;
 652
 653	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M;
 654	req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) &
 655		SEC_BD_W0_C_GRAN_SIZE_19_16_M;
 656
 657	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M;
 658	req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) &
 659		SEC_BD_W0_C_GRAN_SIZE_21_20_M;
 660
 661	/* Writing whole u32 so no need to take care of masking */
 662	req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
 663		((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) &
 664		 SEC_BD_W2_C_GRAN_SIZE_15_0_M);
 665
 666	req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
 667	req->w1 |= SEC_BD_W1_ADDR_TYPE;
 668
 669	el->sgl_in = sgl_in;
 670
 671	ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in,
 672					n_ents_in, info, gfp);
 673	if (ret)
 674		goto err_free_el;
 675
 676	req->data_addr_lo = lower_32_bits(el->dma_in);
 677	req->data_addr_hi = upper_32_bits(el->dma_in);
 678
 679	if (different_dest) {
 680		el->sgl_out = sgl_out;
 681		ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out,
 682						el->sgl_out,
 683						n_ents_out, info, gfp);
 684		if (ret)
 685			goto err_free_hw_sgl_in;
 686
 687		req->w0 |= SEC_BD_W0_DE;
 688		req->cipher_destin_addr_lo = lower_32_bits(el->dma_out);
 689		req->cipher_destin_addr_hi = upper_32_bits(el->dma_out);
 690
 691	} else {
 692		req->w0 &= ~SEC_BD_W0_DE;
 693		req->cipher_destin_addr_lo = lower_32_bits(el->dma_in);
 694		req->cipher_destin_addr_hi = upper_32_bits(el->dma_in);
 695	}
 696
 697	return el;
 698
 699err_free_hw_sgl_in:
 700	sec_free_hw_sgl(el->in, el->dma_in, info);
 701err_free_el:
 702	kfree(el);
 703
 704	return ERR_PTR(ret);
 705}
 706
 707static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
 708				   bool encrypt)
 709{
 710	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
 711	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
 712	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 713	struct sec_queue *queue = ctx->queue;
 714	struct sec_request *sec_req = skcipher_request_ctx(skreq);
 715	struct sec_dev_info *info = queue->dev_info;
 716	int i, ret, steps;
 717	size_t *split_sizes;
 718	struct scatterlist **splits_in;
 719	struct scatterlist **splits_out = NULL;
 720	int *splits_in_nents;
 721	int *splits_out_nents = NULL;
 722	struct sec_request_el *el, *temp;
 723	bool split = skreq->src != skreq->dst;
 724	gfp_t gfp = skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
 725
 726	mutex_init(&sec_req->lock);
 727	sec_req->req_base = &skreq->base;
 728	sec_req->err = 0;
 729	/* SGL mapping out here to allow us to break it up as necessary */
 730	sec_req->len_in = sg_nents(skreq->src);
 731
 732	ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes,
 733						 &steps, gfp);
 734	if (ret)
 735		return ret;
 736	sec_req->num_elements = steps;
 737	ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in,
 738				   &splits_in_nents, sec_req->len_in,
 739				   info->dev, gfp);
 740	if (ret)
 741		goto err_free_split_sizes;
 742
 743	if (split) {
 744		sec_req->len_out = sg_nents(skreq->dst);
 745		ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps,
 746					   &splits_out, &splits_out_nents,
 747					   sec_req->len_out, info->dev, gfp);
 748		if (ret)
 749			goto err_unmap_in_sg;
 750	}
 751	/* Shared info stored in seq_req - applies to all BDs */
 752	sec_req->tfm_ctx = ctx;
 753	sec_req->cb = sec_skcipher_alg_callback;
 754	INIT_LIST_HEAD(&sec_req->elements);
 755
 756	/*
 757	 * Future optimization.
 758	 * In the chaining case we can't use a dma pool bounce buffer
 759	 * but in the case where we know there is no chaining we can
 760	 */
 761	if (crypto_skcipher_ivsize(atfm)) {
 762		sec_req->dma_iv = dma_map_single(info->dev, skreq->iv,
 763						 crypto_skcipher_ivsize(atfm),
 764						 DMA_TO_DEVICE);
 765		if (dma_mapping_error(info->dev, sec_req->dma_iv)) {
 766			ret = -ENOMEM;
 767			goto err_unmap_out_sg;
 768		}
 769	}
 770
 771	/* Set them all up then queue - cleaner error handling. */
 772	for (i = 0; i < steps; i++) {
 773		el = sec_alg_alloc_and_fill_el(&ctx->req_template,
 774					       encrypt ? 1 : 0,
 775					       split_sizes[i],
 776					       skreq->src != skreq->dst,
 777					       splits_in[i], splits_in_nents[i],
 778					       split ? splits_out[i] : NULL,
 779					       split ? splits_out_nents[i] : 0,
 780					       info, gfp);
 781		if (IS_ERR(el)) {
 782			ret = PTR_ERR(el);
 783			goto err_free_elements;
 784		}
 785		el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
 786		el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
 787		el->sec_req = sec_req;
 788		list_add_tail(&el->head, &sec_req->elements);
 789	}
 790
 791	/*
 792	 * Only attempt to queue if the whole lot can fit in the queue -
 793	 * we can't successfully cleanup after a partial queing so this
 794	 * must succeed or fail atomically.
 795	 *
 796	 * Big hammer test of both software and hardware queues - could be
 797	 * more refined but this is unlikely to happen so no need.
 798	 */
 799
 800	/* Grab a big lock for a long time to avoid concurrency issues */
 801	spin_lock_bh(&queue->queuelock);
 802
 803	/*
 804	 * Can go on to queue if we have space in either:
 805	 * 1) The hardware queue and no software queue
 806	 * 2) The software queue
 807	 * AND there is nothing in the backlog.  If there is backlog we
 808	 * have to only queue to the backlog queue and return busy.
 809	 */
 810	if ((!sec_queue_can_enqueue(queue, steps) &&
 811	     (!queue->havesoftqueue ||
 812	      kfifo_avail(&queue->softqueue) > steps)) ||
 813	    !list_empty(&ctx->backlog)) {
 814		ret = -EBUSY;
 815		if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 816			list_add_tail(&sec_req->backlog_head, &ctx->backlog);
 817			spin_unlock_bh(&queue->queuelock);
 818			goto out;
 819		}
 820
 821		spin_unlock_bh(&queue->queuelock);
 822		goto err_free_elements;
 823	}
 824	ret = sec_send_request(sec_req, queue);
 825	spin_unlock_bh(&queue->queuelock);
 826	if (ret)
 827		goto err_free_elements;
 828
 829	ret = -EINPROGRESS;
 830out:
 831	/* Cleanup - all elements in pointer arrays have been copied */
 832	kfree(splits_in_nents);
 833	kfree(splits_in);
 834	kfree(splits_out_nents);
 835	kfree(splits_out);
 836	kfree(split_sizes);
 837	return ret;
 838
 839err_free_elements:
 840	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
 841		list_del(&el->head);
 842		sec_alg_free_el(el, info);
 843	}
 844	if (crypto_skcipher_ivsize(atfm))
 845		dma_unmap_single(info->dev, sec_req->dma_iv,
 846				 crypto_skcipher_ivsize(atfm),
 847				 DMA_BIDIRECTIONAL);
 848err_unmap_out_sg:
 849	if (split)
 850		sec_unmap_sg_on_err(skreq->dst, steps, splits_out,
 851				    splits_out_nents, sec_req->len_out,
 852				    info->dev);
 853err_unmap_in_sg:
 854	sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents,
 855			    sec_req->len_in, info->dev);
 856err_free_split_sizes:
 857	kfree(split_sizes);
 858
 859	return ret;
 860}
 861
 862static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
 863{
 864	return sec_alg_skcipher_crypto(req, true);
 865}
 866
 867static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
 868{
 869	return sec_alg_skcipher_crypto(req, false);
 870}
 871
 872static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
 873{
 874	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 875
 876	mutex_init(&ctx->lock);
 877	INIT_LIST_HEAD(&ctx->backlog);
 878	crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request));
 879
 880	ctx->queue = sec_queue_alloc_start_safe();
 881	if (IS_ERR(ctx->queue))
 882		return PTR_ERR(ctx->queue);
 883
 884	spin_lock_init(&ctx->queue->queuelock);
 885	ctx->queue->havesoftqueue = false;
 886
 887	return 0;
 888}
 889
 890static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
 891{
 892	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 893	struct device *dev = ctx->queue->dev_info->dev;
 894
 895	if (ctx->key) {
 896		memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY);
 897		dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key,
 898				  ctx->pkey);
 899	}
 900	sec_queue_stop_release(ctx->queue);
 901}
 902
 903static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
 904{
 905	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 906	int ret;
 907
 908	ret = sec_alg_skcipher_init(tfm);
 909	if (ret)
 910		return ret;
 911
 912	INIT_KFIFO(ctx->queue->softqueue);
 913	ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL);
 914	if (ret) {
 915		sec_alg_skcipher_exit(tfm);
 916		return ret;
 917	}
 918	ctx->queue->havesoftqueue = true;
 919
 920	return 0;
 921}
 922
 923static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm)
 924{
 925	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 926
 927	kfifo_free(&ctx->queue->softqueue);
 928	sec_alg_skcipher_exit(tfm);
 929}
 930
 931static struct skcipher_alg sec_algs[] = {
 932	{
 933		.base = {
 934			.cra_name = "ecb(aes)",
 935			.cra_driver_name = "hisi_sec_aes_ecb",
 936			.cra_priority = 4001,
 937			.cra_flags = CRYPTO_ALG_ASYNC |
 938				     CRYPTO_ALG_ALLOCATES_MEMORY,
 939			.cra_blocksize = AES_BLOCK_SIZE,
 940			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 941			.cra_alignmask = 0,
 942			.cra_module = THIS_MODULE,
 943		},
 944		.init = sec_alg_skcipher_init,
 945		.exit = sec_alg_skcipher_exit,
 946		.setkey = sec_alg_skcipher_setkey_aes_ecb,
 947		.decrypt = sec_alg_skcipher_decrypt,
 948		.encrypt = sec_alg_skcipher_encrypt,
 949		.min_keysize = AES_MIN_KEY_SIZE,
 950		.max_keysize = AES_MAX_KEY_SIZE,
 951		.ivsize = 0,
 952	}, {
 953		.base = {
 954			.cra_name = "cbc(aes)",
 955			.cra_driver_name = "hisi_sec_aes_cbc",
 956			.cra_priority = 4001,
 957			.cra_flags = CRYPTO_ALG_ASYNC |
 958				     CRYPTO_ALG_ALLOCATES_MEMORY,
 959			.cra_blocksize = AES_BLOCK_SIZE,
 960			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 961			.cra_alignmask = 0,
 962			.cra_module = THIS_MODULE,
 963		},
 964		.init = sec_alg_skcipher_init_with_queue,
 965		.exit = sec_alg_skcipher_exit_with_queue,
 966		.setkey = sec_alg_skcipher_setkey_aes_cbc,
 967		.decrypt = sec_alg_skcipher_decrypt,
 968		.encrypt = sec_alg_skcipher_encrypt,
 969		.min_keysize = AES_MIN_KEY_SIZE,
 970		.max_keysize = AES_MAX_KEY_SIZE,
 971		.ivsize = AES_BLOCK_SIZE,
 972	}, {
 973		.base = {
 974			.cra_name = "ctr(aes)",
 975			.cra_driver_name = "hisi_sec_aes_ctr",
 976			.cra_priority = 4001,
 977			.cra_flags = CRYPTO_ALG_ASYNC |
 978				     CRYPTO_ALG_ALLOCATES_MEMORY,
 979			.cra_blocksize = AES_BLOCK_SIZE,
 980			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 981			.cra_alignmask = 0,
 982			.cra_module = THIS_MODULE,
 983		},
 984		.init = sec_alg_skcipher_init_with_queue,
 985		.exit = sec_alg_skcipher_exit_with_queue,
 986		.setkey = sec_alg_skcipher_setkey_aes_ctr,
 987		.decrypt = sec_alg_skcipher_decrypt,
 988		.encrypt = sec_alg_skcipher_encrypt,
 989		.min_keysize = AES_MIN_KEY_SIZE,
 990		.max_keysize = AES_MAX_KEY_SIZE,
 991		.ivsize = AES_BLOCK_SIZE,
 992	}, {
 993		.base = {
 994			.cra_name = "xts(aes)",
 995			.cra_driver_name = "hisi_sec_aes_xts",
 996			.cra_priority = 4001,
 997			.cra_flags = CRYPTO_ALG_ASYNC |
 998				     CRYPTO_ALG_ALLOCATES_MEMORY,
 999			.cra_blocksize = AES_BLOCK_SIZE,
1000			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1001			.cra_alignmask = 0,
1002			.cra_module = THIS_MODULE,
1003		},
1004		.init = sec_alg_skcipher_init,
1005		.exit = sec_alg_skcipher_exit,
1006		.setkey = sec_alg_skcipher_setkey_aes_xts,
1007		.decrypt = sec_alg_skcipher_decrypt,
1008		.encrypt = sec_alg_skcipher_encrypt,
1009		.min_keysize = 2 * AES_MIN_KEY_SIZE,
1010		.max_keysize = 2 * AES_MAX_KEY_SIZE,
1011		.ivsize = AES_BLOCK_SIZE,
1012	}, {
1013	/* Unable to find any test vectors so untested */
1014		.base = {
1015			.cra_name = "ecb(des)",
1016			.cra_driver_name = "hisi_sec_des_ecb",
1017			.cra_priority = 4001,
1018			.cra_flags = CRYPTO_ALG_ASYNC |
1019				     CRYPTO_ALG_ALLOCATES_MEMORY,
1020			.cra_blocksize = DES_BLOCK_SIZE,
1021			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1022			.cra_alignmask = 0,
1023			.cra_module = THIS_MODULE,
1024		},
1025		.init = sec_alg_skcipher_init,
1026		.exit = sec_alg_skcipher_exit,
1027		.setkey = sec_alg_skcipher_setkey_des_ecb,
1028		.decrypt = sec_alg_skcipher_decrypt,
1029		.encrypt = sec_alg_skcipher_encrypt,
1030		.min_keysize = DES_KEY_SIZE,
1031		.max_keysize = DES_KEY_SIZE,
1032		.ivsize = 0,
1033	}, {
1034		.base = {
1035			.cra_name = "cbc(des)",
1036			.cra_driver_name = "hisi_sec_des_cbc",
1037			.cra_priority = 4001,
1038			.cra_flags = CRYPTO_ALG_ASYNC |
1039				     CRYPTO_ALG_ALLOCATES_MEMORY,
1040			.cra_blocksize = DES_BLOCK_SIZE,
1041			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1042			.cra_alignmask = 0,
1043			.cra_module = THIS_MODULE,
1044		},
1045		.init = sec_alg_skcipher_init_with_queue,
1046		.exit = sec_alg_skcipher_exit_with_queue,
1047		.setkey = sec_alg_skcipher_setkey_des_cbc,
1048		.decrypt = sec_alg_skcipher_decrypt,
1049		.encrypt = sec_alg_skcipher_encrypt,
1050		.min_keysize = DES_KEY_SIZE,
1051		.max_keysize = DES_KEY_SIZE,
1052		.ivsize = DES_BLOCK_SIZE,
1053	}, {
1054		.base = {
1055			.cra_name = "cbc(des3_ede)",
1056			.cra_driver_name = "hisi_sec_3des_cbc",
1057			.cra_priority = 4001,
1058			.cra_flags = CRYPTO_ALG_ASYNC |
1059				     CRYPTO_ALG_ALLOCATES_MEMORY,
1060			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1061			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1062			.cra_alignmask = 0,
1063			.cra_module = THIS_MODULE,
1064		},
1065		.init = sec_alg_skcipher_init_with_queue,
1066		.exit = sec_alg_skcipher_exit_with_queue,
1067		.setkey = sec_alg_skcipher_setkey_3des_cbc,
1068		.decrypt = sec_alg_skcipher_decrypt,
1069		.encrypt = sec_alg_skcipher_encrypt,
1070		.min_keysize = DES3_EDE_KEY_SIZE,
1071		.max_keysize = DES3_EDE_KEY_SIZE,
1072		.ivsize = DES3_EDE_BLOCK_SIZE,
1073	}, {
1074		.base = {
1075			.cra_name = "ecb(des3_ede)",
1076			.cra_driver_name = "hisi_sec_3des_ecb",
1077			.cra_priority = 4001,
1078			.cra_flags = CRYPTO_ALG_ASYNC |
1079				     CRYPTO_ALG_ALLOCATES_MEMORY,
1080			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1081			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1082			.cra_alignmask = 0,
1083			.cra_module = THIS_MODULE,
1084		},
1085		.init = sec_alg_skcipher_init,
1086		.exit = sec_alg_skcipher_exit,
1087		.setkey = sec_alg_skcipher_setkey_3des_ecb,
1088		.decrypt = sec_alg_skcipher_decrypt,
1089		.encrypt = sec_alg_skcipher_encrypt,
1090		.min_keysize = DES3_EDE_KEY_SIZE,
1091		.max_keysize = DES3_EDE_KEY_SIZE,
1092		.ivsize = 0,
1093	}
1094};
1095
1096int sec_algs_register(void)
1097{
1098	int ret = 0;
1099
1100	mutex_lock(&algs_lock);
1101	if (++active_devs != 1)
1102		goto unlock;
1103
1104	ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1105	if (ret)
1106		--active_devs;
1107unlock:
1108	mutex_unlock(&algs_lock);
1109
1110	return ret;
1111}
1112
1113void sec_algs_unregister(void)
1114{
1115	mutex_lock(&algs_lock);
1116	if (--active_devs != 0)
1117		goto unlock;
1118	crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1119
1120unlock:
1121	mutex_unlock(&algs_lock);
1122}