1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * AMD Cryptographic Coprocessor (CCP) driver
   4 *
   5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
   6 *
   7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8 * Author: Gary R Hook <gary.hook@amd.com>
   9 */
  10
  11#include <linux/dma-mapping.h>
  12#include <linux/module.h>
  13#include <linux/kernel.h>
  14#include <linux/interrupt.h>
  15#include <crypto/scatterwalk.h>
  16#include <crypto/des.h>
  17#include <linux/ccp.h>
  18
  19#include "ccp-dev.h"
  20
  21/* SHA initial context values */
  22static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23	cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24	cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25	cpu_to_be32(SHA1_H4),
  26};
  27
  28static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  29	cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  30	cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  31	cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  32	cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  33};
  34
  35static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  36	cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  37	cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  38	cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  39	cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  40};
  41
  42static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  43	cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  44	cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  45	cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  46	cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  47};
  48
  49static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  50	cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  51	cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  52	cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  53	cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  54};
  55
  56#define	CCP_NEW_JOBID(ccp)	((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  57					ccp_gen_jobid(ccp) : 0)
  58
  59static u32 ccp_gen_jobid(struct ccp_device *ccp)
  60{
  61	return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  62}
  63
  64static void ccp_sg_free(struct ccp_sg_workarea *wa)
  65{
  66	if (wa->dma_count)
  67		dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  68
  69	wa->dma_count = 0;
  70}
  71
  72static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  73				struct scatterlist *sg, u64 len,
  74				enum dma_data_direction dma_dir)
  75{
  76	memset(wa, 0, sizeof(*wa));
  77
  78	wa->sg = sg;
  79	if (!sg)
  80		return 0;
  81
  82	wa->nents = sg_nents_for_len(sg, len);
  83	if (wa->nents < 0)
  84		return wa->nents;
  85
  86	wa->bytes_left = len;
  87	wa->sg_used = 0;
  88
  89	if (len == 0)
  90		return 0;
  91
  92	if (dma_dir == DMA_NONE)
  93		return 0;
  94
  95	wa->dma_sg = sg;
  96	wa->dma_sg_head = sg;
  97	wa->dma_dev = dev;
  98	wa->dma_dir = dma_dir;
  99	wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 100	if (!wa->dma_count)
 101		return -ENOMEM;
 102
 103	return 0;
 104}
 105
 106static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 107{
 108	unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 109	unsigned int sg_combined_len = 0;
 110
 111	if (!wa->sg)
 112		return;
 113
 114	wa->sg_used += nbytes;
 115	wa->bytes_left -= nbytes;
 116	if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
 117		/* Advance to the next DMA scatterlist entry */
 118		wa->dma_sg = sg_next(wa->dma_sg);
 119
 120		/* In the case that the DMA mapped scatterlist has entries
 121		 * that have been merged, the non-DMA mapped scatterlist
 122		 * must be advanced multiple times for each merged entry.
 123		 * This ensures that the current non-DMA mapped entry
 124		 * corresponds to the current DMA mapped entry.
 125		 */
 126		do {
 127			sg_combined_len += wa->sg->length;
 128			wa->sg = sg_next(wa->sg);
 129		} while (wa->sg_used > sg_combined_len);
 130
 131		wa->sg_used = 0;
 132	}
 133}
 134
 135static void ccp_dm_free(struct ccp_dm_workarea *wa)
 136{
 137	if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 138		if (wa->address)
 139			dma_pool_free(wa->dma_pool, wa->address,
 140				      wa->dma.address);
 141	} else {
 142		if (wa->dma.address)
 143			dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 144					 wa->dma.dir);
 145		kfree(wa->address);
 146	}
 147
 148	wa->address = NULL;
 149	wa->dma.address = 0;
 150}
 151
 152static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 153				struct ccp_cmd_queue *cmd_q,
 154				unsigned int len,
 155				enum dma_data_direction dir)
 156{
 157	memset(wa, 0, sizeof(*wa));
 158
 159	if (!len)
 160		return 0;
 161
 162	wa->dev = cmd_q->ccp->dev;
 163	wa->length = len;
 164
 165	if (len <= CCP_DMAPOOL_MAX_SIZE) {
 166		wa->dma_pool = cmd_q->dma_pool;
 167
 168		wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
 169					     &wa->dma.address);
 170		if (!wa->address)
 171			return -ENOMEM;
 172
 173		wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 174
 175	} else {
 176		wa->address = kzalloc(len, GFP_KERNEL);
 177		if (!wa->address)
 178			return -ENOMEM;
 179
 180		wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 181						 dir);
 182		if (dma_mapping_error(wa->dev, wa->dma.address))
 183			return -ENOMEM;
 184
 185		wa->dma.length = len;
 186	}
 187	wa->dma.dir = dir;
 188
 189	return 0;
 190}
 191
 192static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 193			   struct scatterlist *sg, unsigned int sg_offset,
 194			   unsigned int len)
 195{
 196	WARN_ON(!wa->address);
 197
 198	if (len > (wa->length - wa_offset))
 199		return -EINVAL;
 200
 201	scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 202				 0);
 203	return 0;
 204}
 205
 206static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 207			    struct scatterlist *sg, unsigned int sg_offset,
 208			    unsigned int len)
 209{
 210	WARN_ON(!wa->address);
 211
 212	scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 213				 1);
 214}
 215
 216static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 217				   unsigned int wa_offset,
 218				   struct scatterlist *sg,
 219				   unsigned int sg_offset,
 220				   unsigned int len)
 221{
 222	u8 *p, *q;
 223	int	rc;
 224
 225	rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 226	if (rc)
 227		return rc;
 228
 229	p = wa->address + wa_offset;
 230	q = p + len - 1;
 231	while (p < q) {
 232		*p = *p ^ *q;
 233		*q = *p ^ *q;
 234		*p = *p ^ *q;
 235		p++;
 236		q--;
 237	}
 238	return 0;
 239}
 240
 241static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 242				    unsigned int wa_offset,
 243				    struct scatterlist *sg,
 244				    unsigned int sg_offset,
 245				    unsigned int len)
 246{
 247	u8 *p, *q;
 248
 249	p = wa->address + wa_offset;
 250	q = p + len - 1;
 251	while (p < q) {
 252		*p = *p ^ *q;
 253		*q = *p ^ *q;
 254		*p = *p ^ *q;
 255		p++;
 256		q--;
 257	}
 258
 259	ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 260}
 261
 262static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 263{
 264	ccp_dm_free(&data->dm_wa);
 265	ccp_sg_free(&data->sg_wa);
 266}
 267
 268static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 269			 struct scatterlist *sg, u64 sg_len,
 270			 unsigned int dm_len,
 271			 enum dma_data_direction dir)
 272{
 273	int ret;
 274
 275	memset(data, 0, sizeof(*data));
 276
 277	ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 278				   dir);
 279	if (ret)
 280		goto e_err;
 281
 282	ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 283	if (ret)
 284		goto e_err;
 285
 286	return 0;
 287
 288e_err:
 289	ccp_free_data(data, cmd_q);
 290
 291	return ret;
 292}
 293
 294static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 295{
 296	struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 297	struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 298	unsigned int buf_count, nbytes;
 299
 300	/* Clear the buffer if setting it */
 301	if (!from)
 302		memset(dm_wa->address, 0, dm_wa->length);
 303
 304	if (!sg_wa->sg)
 305		return 0;
 306
 307	/* Perform the copy operation
 308	 *   nbytes will always be <= UINT_MAX because dm_wa->length is
 309	 *   an unsigned int
 310	 */
 311	nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 312	scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 313				 nbytes, from);
 314
 315	/* Update the structures and generate the count */
 316	buf_count = 0;
 317	while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 318		nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
 319			     dm_wa->length - buf_count);
 320		nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 321
 322		buf_count += nbytes;
 323		ccp_update_sg_workarea(sg_wa, nbytes);
 324	}
 325
 326	return buf_count;
 327}
 328
 329static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 330{
 331	return ccp_queue_buf(data, 0);
 332}
 333
 334static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 335{
 336	return ccp_queue_buf(data, 1);
 337}
 338
 339static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 340			     struct ccp_op *op, unsigned int block_size,
 341			     bool blocksize_op)
 342{
 343	unsigned int sg_src_len, sg_dst_len, op_len;
 344
 345	/* The CCP can only DMA from/to one address each per operation. This
 346	 * requires that we find the smallest DMA area between the source
 347	 * and destination. The resulting len values will always be <= UINT_MAX
 348	 * because the dma length is an unsigned int.
 349	 */
 350	sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
 351	sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 352
 353	if (dst) {
 354		sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
 355		sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 356		op_len = min(sg_src_len, sg_dst_len);
 357	} else {
 358		op_len = sg_src_len;
 359	}
 360
 361	/* The data operation length will be at least block_size in length
 362	 * or the smaller of available sg room remaining for the source or
 363	 * the destination
 364	 */
 365	op_len = max(op_len, block_size);
 366
 367	/* Unless we have to buffer data, there's no reason to wait */
 368	op->soc = 0;
 369
 370	if (sg_src_len < block_size) {
 371		/* Not enough data in the sg element, so it
 372		 * needs to be buffered into a blocksize chunk
 373		 */
 374		int cp_len = ccp_fill_queue_buf(src);
 375
 376		op->soc = 1;
 377		op->src.u.dma.address = src->dm_wa.dma.address;
 378		op->src.u.dma.offset = 0;
 379		op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 380	} else {
 381		/* Enough data in the sg element, but we need to
 382		 * adjust for any previously copied data
 383		 */
 384		op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
 385		op->src.u.dma.offset = src->sg_wa.sg_used;
 386		op->src.u.dma.length = op_len & ~(block_size - 1);
 387
 388		ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 389	}
 390
 391	if (dst) {
 392		if (sg_dst_len < block_size) {
 393			/* Not enough room in the sg element or we're on the
 394			 * last piece of data (when using padding), so the
 395			 * output needs to be buffered into a blocksize chunk
 396			 */
 397			op->soc = 1;
 398			op->dst.u.dma.address = dst->dm_wa.dma.address;
 399			op->dst.u.dma.offset = 0;
 400			op->dst.u.dma.length = op->src.u.dma.length;
 401		} else {
 402			/* Enough room in the sg element, but we need to
 403			 * adjust for any previously used area
 404			 */
 405			op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
 406			op->dst.u.dma.offset = dst->sg_wa.sg_used;
 407			op->dst.u.dma.length = op->src.u.dma.length;
 408		}
 409	}
 410}
 411
 412static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 413			     struct ccp_op *op)
 414{
 415	op->init = 0;
 416
 417	if (dst) {
 418		if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 419			ccp_empty_queue_buf(dst);
 420		else
 421			ccp_update_sg_workarea(&dst->sg_wa,
 422					       op->dst.u.dma.length);
 423	}
 424}
 425
 426static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 427			       struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 428			       u32 byte_swap, bool from)
 429{
 430	struct ccp_op op;
 431
 432	memset(&op, 0, sizeof(op));
 433
 434	op.cmd_q = cmd_q;
 435	op.jobid = jobid;
 436	op.eom = 1;
 437
 438	if (from) {
 439		op.soc = 1;
 440		op.src.type = CCP_MEMTYPE_SB;
 441		op.src.u.sb = sb;
 442		op.dst.type = CCP_MEMTYPE_SYSTEM;
 443		op.dst.u.dma.address = wa->dma.address;
 444		op.dst.u.dma.length = wa->length;
 445	} else {
 446		op.src.type = CCP_MEMTYPE_SYSTEM;
 447		op.src.u.dma.address = wa->dma.address;
 448		op.src.u.dma.length = wa->length;
 449		op.dst.type = CCP_MEMTYPE_SB;
 450		op.dst.u.sb = sb;
 451	}
 452
 453	op.u.passthru.byte_swap = byte_swap;
 454
 455	return cmd_q->ccp->vdata->perform->passthru(&op);
 456}
 457
 458static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 459			  struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 460			  u32 byte_swap)
 461{
 462	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 463}
 464
 465static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 466			    struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 467			    u32 byte_swap)
 468{
 469	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 470}
 471
 472static noinline_for_stack int
 473ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 474{
 475	struct ccp_aes_engine *aes = &cmd->u.aes;
 476	struct ccp_dm_workarea key, ctx;
 477	struct ccp_data src;
 478	struct ccp_op op;
 479	unsigned int dm_offset;
 480	int ret;
 481
 482	if (!((aes->key_len == AES_KEYSIZE_128) ||
 483	      (aes->key_len == AES_KEYSIZE_192) ||
 484	      (aes->key_len == AES_KEYSIZE_256)))
 485		return -EINVAL;
 486
 487	if (aes->src_len & (AES_BLOCK_SIZE - 1))
 488		return -EINVAL;
 489
 490	if (aes->iv_len != AES_BLOCK_SIZE)
 491		return -EINVAL;
 492
 493	if (!aes->key || !aes->iv || !aes->src)
 494		return -EINVAL;
 495
 496	if (aes->cmac_final) {
 497		if (aes->cmac_key_len != AES_BLOCK_SIZE)
 498			return -EINVAL;
 499
 500		if (!aes->cmac_key)
 501			return -EINVAL;
 502	}
 503
 504	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 505	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 506
 507	ret = -EIO;
 508	memset(&op, 0, sizeof(op));
 509	op.cmd_q = cmd_q;
 510	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 511	op.sb_key = cmd_q->sb_key;
 512	op.sb_ctx = cmd_q->sb_ctx;
 513	op.init = 1;
 514	op.u.aes.type = aes->type;
 515	op.u.aes.mode = aes->mode;
 516	op.u.aes.action = aes->action;
 517
 518	/* All supported key sizes fit in a single (32-byte) SB entry
 519	 * and must be in little endian format. Use the 256-bit byte
 520	 * swap passthru option to convert from big endian to little
 521	 * endian.
 522	 */
 523	ret = ccp_init_dm_workarea(&key, cmd_q,
 524				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 525				   DMA_TO_DEVICE);
 526	if (ret)
 527		return ret;
 528
 529	dm_offset = CCP_SB_BYTES - aes->key_len;
 530	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 531	if (ret)
 532		goto e_key;
 533	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 534			     CCP_PASSTHRU_BYTESWAP_256BIT);
 535	if (ret) {
 536		cmd->engine_error = cmd_q->cmd_error;
 537		goto e_key;
 538	}
 539
 540	/* The AES context fits in a single (32-byte) SB entry and
 541	 * must be in little endian format. Use the 256-bit byte swap
 542	 * passthru option to convert from big endian to little endian.
 543	 */
 544	ret = ccp_init_dm_workarea(&ctx, cmd_q,
 545				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 546				   DMA_BIDIRECTIONAL);
 547	if (ret)
 548		goto e_key;
 549
 550	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 551	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 552	if (ret)
 553		goto e_ctx;
 554	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 555			     CCP_PASSTHRU_BYTESWAP_256BIT);
 556	if (ret) {
 557		cmd->engine_error = cmd_q->cmd_error;
 558		goto e_ctx;
 559	}
 560
 561	/* Send data to the CCP AES engine */
 562	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 563			    AES_BLOCK_SIZE, DMA_TO_DEVICE);
 564	if (ret)
 565		goto e_ctx;
 566
 567	while (src.sg_wa.bytes_left) {
 568		ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 569		if (aes->cmac_final && !src.sg_wa.bytes_left) {
 570			op.eom = 1;
 571
 572			/* Push the K1/K2 key to the CCP now */
 573			ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 574					       op.sb_ctx,
 575					       CCP_PASSTHRU_BYTESWAP_256BIT);
 576			if (ret) {
 577				cmd->engine_error = cmd_q->cmd_error;
 578				goto e_src;
 579			}
 580
 581			ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 582					      aes->cmac_key_len);
 583			if (ret)
 584				goto e_src;
 585			ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 586					     CCP_PASSTHRU_BYTESWAP_256BIT);
 587			if (ret) {
 588				cmd->engine_error = cmd_q->cmd_error;
 589				goto e_src;
 590			}
 591		}
 592
 593		ret = cmd_q->ccp->vdata->perform->aes(&op);
 594		if (ret) {
 595			cmd->engine_error = cmd_q->cmd_error;
 596			goto e_src;
 597		}
 598
 599		ccp_process_data(&src, NULL, &op);
 600	}
 601
 602	/* Retrieve the AES context - convert from LE to BE using
 603	 * 32-byte (256-bit) byteswapping
 604	 */
 605	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 606			       CCP_PASSTHRU_BYTESWAP_256BIT);
 607	if (ret) {
 608		cmd->engine_error = cmd_q->cmd_error;
 609		goto e_src;
 610	}
 611
 612	/* ...but we only need AES_BLOCK_SIZE bytes */
 613	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 614	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 615
 616e_src:
 617	ccp_free_data(&src, cmd_q);
 618
 619e_ctx:
 620	ccp_dm_free(&ctx);
 621
 622e_key:
 623	ccp_dm_free(&key);
 624
 625	return ret;
 626}
 627
 628static noinline_for_stack int
 629ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 630{
 631	struct ccp_aes_engine *aes = &cmd->u.aes;
 632	struct ccp_dm_workarea key, ctx, final_wa, tag;
 633	struct ccp_data src, dst;
 634	struct ccp_data aad;
 635	struct ccp_op op;
 636	unsigned int dm_offset;
 637	unsigned int authsize;
 638	unsigned int jobid;
 639	unsigned int ilen;
 640	bool in_place = true; /* Default value */
 641	__be64 *final;
 642	int ret;
 643
 644	struct scatterlist *p_inp, sg_inp[2];
 645	struct scatterlist *p_tag, sg_tag[2];
 646	struct scatterlist *p_outp, sg_outp[2];
 647	struct scatterlist *p_aad;
 648
 649	if (!aes->iv)
 650		return -EINVAL;
 651
 652	if (!((aes->key_len == AES_KEYSIZE_128) ||
 653		(aes->key_len == AES_KEYSIZE_192) ||
 654		(aes->key_len == AES_KEYSIZE_256)))
 655		return -EINVAL;
 656
 657	if (!aes->key) /* Gotta have a key SGL */
 658		return -EINVAL;
 659
 660	/* Zero defaults to 16 bytes, the maximum size */
 661	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 662	switch (authsize) {
 663	case 16:
 664	case 15:
 665	case 14:
 666	case 13:
 667	case 12:
 668	case 8:
 669	case 4:
 670		break;
 671	default:
 672		return -EINVAL;
 673	}
 674
 675	/* First, decompose the source buffer into AAD & PT,
 676	 * and the destination buffer into AAD, CT & tag, or
 677	 * the input into CT & tag.
 678	 * It is expected that the input and output SGs will
 679	 * be valid, even if the AAD and input lengths are 0.
 680	 */
 681	p_aad = aes->src;
 682	p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 683	p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 684	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 685		ilen = aes->src_len;
 686		p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 687	} else {
 688		/* Input length for decryption includes tag */
 689		ilen = aes->src_len - authsize;
 690		p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 691	}
 692
 693	jobid = CCP_NEW_JOBID(cmd_q->ccp);
 694
 695	memset(&op, 0, sizeof(op));
 696	op.cmd_q = cmd_q;
 697	op.jobid = jobid;
 698	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 699	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 700	op.init = 1;
 701	op.u.aes.type = aes->type;
 702
 703	/* Copy the key to the LSB */
 704	ret = ccp_init_dm_workarea(&key, cmd_q,
 705				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 706				   DMA_TO_DEVICE);
 707	if (ret)
 708		return ret;
 709
 710	dm_offset = CCP_SB_BYTES - aes->key_len;
 711	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 712	if (ret)
 713		goto e_key;
 714	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 715			     CCP_PASSTHRU_BYTESWAP_256BIT);
 716	if (ret) {
 717		cmd->engine_error = cmd_q->cmd_error;
 718		goto e_key;
 719	}
 720
 721	/* Copy the context (IV) to the LSB.
 722	 * There is an assumption here that the IV is 96 bits in length, plus
 723	 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 724	 */
 725	ret = ccp_init_dm_workarea(&ctx, cmd_q,
 726				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 727				   DMA_BIDIRECTIONAL);
 728	if (ret)
 729		goto e_key;
 730
 731	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 732	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 733	if (ret)
 734		goto e_ctx;
 735
 736	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 737			     CCP_PASSTHRU_BYTESWAP_256BIT);
 738	if (ret) {
 739		cmd->engine_error = cmd_q->cmd_error;
 740		goto e_ctx;
 741	}
 742
 743	op.init = 1;
 744	if (aes->aad_len > 0) {
 745		/* Step 1: Run a GHASH over the Additional Authenticated Data */
 746		ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 747				    AES_BLOCK_SIZE,
 748				    DMA_TO_DEVICE);
 749		if (ret)
 750			goto e_ctx;
 751
 752		op.u.aes.mode = CCP_AES_MODE_GHASH;
 753		op.u.aes.action = CCP_AES_GHASHAAD;
 754
 755		while (aad.sg_wa.bytes_left) {
 756			ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 757
 758			ret = cmd_q->ccp->vdata->perform->aes(&op);
 759			if (ret) {
 760				cmd->engine_error = cmd_q->cmd_error;
 761				goto e_aad;
 762			}
 763
 764			ccp_process_data(&aad, NULL, &op);
 765			op.init = 0;
 766		}
 767	}
 768
 769	op.u.aes.mode = CCP_AES_MODE_GCTR;
 770	op.u.aes.action = aes->action;
 771
 772	if (ilen > 0) {
 773		/* Step 2: Run a GCTR over the plaintext */
 774		in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 775
 776		ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 777				    AES_BLOCK_SIZE,
 778				    in_place ? DMA_BIDIRECTIONAL
 779					     : DMA_TO_DEVICE);
 780		if (ret)
 781			goto e_aad;
 782
 783		if (in_place) {
 784			dst = src;
 785		} else {
 786			ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 787					    AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 788			if (ret)
 789				goto e_src;
 790		}
 791
 792		op.soc = 0;
 793		op.eom = 0;
 794		op.init = 1;
 795		while (src.sg_wa.bytes_left) {
 796			ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 797			if (!src.sg_wa.bytes_left) {
 798				unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 799
 800				if (nbytes) {
 801					op.eom = 1;
 802					op.u.aes.size = (nbytes * 8) - 1;
 803				}
 804			}
 805
 806			ret = cmd_q->ccp->vdata->perform->aes(&op);
 807			if (ret) {
 808				cmd->engine_error = cmd_q->cmd_error;
 809				goto e_dst;
 810			}
 811
 812			ccp_process_data(&src, &dst, &op);
 813			op.init = 0;
 814		}
 815	}
 816
 817	/* Step 3: Update the IV portion of the context with the original IV */
 818	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 819			       CCP_PASSTHRU_BYTESWAP_256BIT);
 820	if (ret) {
 821		cmd->engine_error = cmd_q->cmd_error;
 822		goto e_dst;
 823	}
 824
 825	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 826	if (ret)
 827		goto e_dst;
 828
 829	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 830			     CCP_PASSTHRU_BYTESWAP_256BIT);
 831	if (ret) {
 832		cmd->engine_error = cmd_q->cmd_error;
 833		goto e_dst;
 834	}
 835
 836	/* Step 4: Concatenate the lengths of the AAD and source, and
 837	 * hash that 16 byte buffer.
 838	 */
 839	ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 840				   DMA_BIDIRECTIONAL);
 841	if (ret)
 842		goto e_dst;
 843	final = (__be64 *)final_wa.address;
 844	final[0] = cpu_to_be64(aes->aad_len * 8);
 845	final[1] = cpu_to_be64(ilen * 8);
 846
 847	memset(&op, 0, sizeof(op));
 848	op.cmd_q = cmd_q;
 849	op.jobid = jobid;
 850	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 851	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 852	op.init = 1;
 853	op.u.aes.type = aes->type;
 854	op.u.aes.mode = CCP_AES_MODE_GHASH;
 855	op.u.aes.action = CCP_AES_GHASHFINAL;
 856	op.src.type = CCP_MEMTYPE_SYSTEM;
 857	op.src.u.dma.address = final_wa.dma.address;
 858	op.src.u.dma.length = AES_BLOCK_SIZE;
 859	op.dst.type = CCP_MEMTYPE_SYSTEM;
 860	op.dst.u.dma.address = final_wa.dma.address;
 861	op.dst.u.dma.length = AES_BLOCK_SIZE;
 862	op.eom = 1;
 863	op.u.aes.size = 0;
 864	ret = cmd_q->ccp->vdata->perform->aes(&op);
 865	if (ret)
 866		goto e_final_wa;
 867
 868	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 869		/* Put the ciphered tag after the ciphertext. */
 870		ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 871	} else {
 872		/* Does this ciphered tag match the input? */
 873		ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 874					   DMA_BIDIRECTIONAL);
 875		if (ret)
 876			goto e_final_wa;
 877		ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 878		if (ret) {
 879			ccp_dm_free(&tag);
 880			goto e_final_wa;
 881		}
 882
 883		ret = crypto_memneq(tag.address, final_wa.address,
 884				    authsize) ? -EBADMSG : 0;
 885		ccp_dm_free(&tag);
 886	}
 887
 888e_final_wa:
 889	ccp_dm_free(&final_wa);
 890
 891e_dst:
 892	if (ilen > 0 && !in_place)
 893		ccp_free_data(&dst, cmd_q);
 894
 895e_src:
 896	if (ilen > 0)
 897		ccp_free_data(&src, cmd_q);
 898
 899e_aad:
 900	if (aes->aad_len)
 901		ccp_free_data(&aad, cmd_q);
 902
 903e_ctx:
 904	ccp_dm_free(&ctx);
 905
 906e_key:
 907	ccp_dm_free(&key);
 908
 909	return ret;
 910}
 911
 912static noinline_for_stack int
 913ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 914{
 915	struct ccp_aes_engine *aes = &cmd->u.aes;
 916	struct ccp_dm_workarea key, ctx;
 917	struct ccp_data src, dst;
 918	struct ccp_op op;
 919	unsigned int dm_offset;
 920	bool in_place = false;
 921	int ret;
 922
 923	if (!((aes->key_len == AES_KEYSIZE_128) ||
 924	      (aes->key_len == AES_KEYSIZE_192) ||
 925	      (aes->key_len == AES_KEYSIZE_256)))
 926		return -EINVAL;
 927
 928	if (((aes->mode == CCP_AES_MODE_ECB) ||
 929	     (aes->mode == CCP_AES_MODE_CBC)) &&
 930	    (aes->src_len & (AES_BLOCK_SIZE - 1)))
 931		return -EINVAL;
 932
 933	if (!aes->key || !aes->src || !aes->dst)
 934		return -EINVAL;
 935
 936	if (aes->mode != CCP_AES_MODE_ECB) {
 937		if (aes->iv_len != AES_BLOCK_SIZE)
 938			return -EINVAL;
 939
 940		if (!aes->iv)
 941			return -EINVAL;
 942	}
 943
 944	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 945	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 946
 947	ret = -EIO;
 948	memset(&op, 0, sizeof(op));
 949	op.cmd_q = cmd_q;
 950	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 951	op.sb_key = cmd_q->sb_key;
 952	op.sb_ctx = cmd_q->sb_ctx;
 953	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 954	op.u.aes.type = aes->type;
 955	op.u.aes.mode = aes->mode;
 956	op.u.aes.action = aes->action;
 957
 958	/* All supported key sizes fit in a single (32-byte) SB entry
 959	 * and must be in little endian format. Use the 256-bit byte
 960	 * swap passthru option to convert from big endian to little
 961	 * endian.
 962	 */
 963	ret = ccp_init_dm_workarea(&key, cmd_q,
 964				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 965				   DMA_TO_DEVICE);
 966	if (ret)
 967		return ret;
 968
 969	dm_offset = CCP_SB_BYTES - aes->key_len;
 970	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 971	if (ret)
 972		goto e_key;
 973	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 974			     CCP_PASSTHRU_BYTESWAP_256BIT);
 975	if (ret) {
 976		cmd->engine_error = cmd_q->cmd_error;
 977		goto e_key;
 978	}
 979
 980	/* The AES context fits in a single (32-byte) SB entry and
 981	 * must be in little endian format. Use the 256-bit byte swap
 982	 * passthru option to convert from big endian to little endian.
 983	 */
 984	ret = ccp_init_dm_workarea(&ctx, cmd_q,
 985				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 986				   DMA_BIDIRECTIONAL);
 987	if (ret)
 988		goto e_key;
 989
 990	if (aes->mode != CCP_AES_MODE_ECB) {
 991		/* Load the AES context - convert to LE */
 992		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 993		ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 994		if (ret)
 995			goto e_ctx;
 996		ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 997				     CCP_PASSTHRU_BYTESWAP_256BIT);
 998		if (ret) {
 999			cmd->engine_error = cmd_q->cmd_error;
1000			goto e_ctx;
1001		}
1002	}
1003	switch (aes->mode) {
1004	case CCP_AES_MODE_CFB: /* CFB128 only */
1005	case CCP_AES_MODE_CTR:
1006		op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1007		break;
1008	default:
1009		op.u.aes.size = 0;
1010	}
1011
1012	/* Prepare the input and output data workareas. For in-place
1013	 * operations we need to set the dma direction to BIDIRECTIONAL
1014	 * and copy the src workarea to the dst workarea.
1015	 */
1016	if (sg_virt(aes->src) == sg_virt(aes->dst))
1017		in_place = true;
1018
1019	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1020			    AES_BLOCK_SIZE,
1021			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1022	if (ret)
1023		goto e_ctx;
1024
1025	if (in_place) {
1026		dst = src;
1027	} else {
1028		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1029				    AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1030		if (ret)
1031			goto e_src;
1032	}
1033
1034	/* Send data to the CCP AES engine */
1035	while (src.sg_wa.bytes_left) {
1036		ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1037		if (!src.sg_wa.bytes_left) {
1038			op.eom = 1;
1039
1040			/* Since we don't retrieve the AES context in ECB
1041			 * mode we have to wait for the operation to complete
1042			 * on the last piece of data
1043			 */
1044			if (aes->mode == CCP_AES_MODE_ECB)
1045				op.soc = 1;
1046		}
1047
1048		ret = cmd_q->ccp->vdata->perform->aes(&op);
1049		if (ret) {
1050			cmd->engine_error = cmd_q->cmd_error;
1051			goto e_dst;
1052		}
1053
1054		ccp_process_data(&src, &dst, &op);
1055	}
1056
1057	if (aes->mode != CCP_AES_MODE_ECB) {
1058		/* Retrieve the AES context - convert from LE to BE using
1059		 * 32-byte (256-bit) byteswapping
1060		 */
1061		ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1062				       CCP_PASSTHRU_BYTESWAP_256BIT);
1063		if (ret) {
1064			cmd->engine_error = cmd_q->cmd_error;
1065			goto e_dst;
1066		}
1067
1068		/* ...but we only need AES_BLOCK_SIZE bytes */
1069		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1070		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1071	}
1072
1073e_dst:
1074	if (!in_place)
1075		ccp_free_data(&dst, cmd_q);
1076
1077e_src:
1078	ccp_free_data(&src, cmd_q);
1079
1080e_ctx:
1081	ccp_dm_free(&ctx);
1082
1083e_key:
1084	ccp_dm_free(&key);
1085
1086	return ret;
1087}
1088
1089static noinline_for_stack int
1090ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1091{
1092	struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1093	struct ccp_dm_workarea key, ctx;
1094	struct ccp_data src, dst;
1095	struct ccp_op op;
1096	unsigned int unit_size, dm_offset;
1097	bool in_place = false;
1098	unsigned int sb_count;
1099	enum ccp_aes_type aestype;
1100	int ret;
1101
1102	switch (xts->unit_size) {
1103	case CCP_XTS_AES_UNIT_SIZE_16:
1104		unit_size = 16;
1105		break;
1106	case CCP_XTS_AES_UNIT_SIZE_512:
1107		unit_size = 512;
1108		break;
1109	case CCP_XTS_AES_UNIT_SIZE_1024:
1110		unit_size = 1024;
1111		break;
1112	case CCP_XTS_AES_UNIT_SIZE_2048:
1113		unit_size = 2048;
1114		break;
1115	case CCP_XTS_AES_UNIT_SIZE_4096:
1116		unit_size = 4096;
1117		break;
1118
1119	default:
1120		return -EINVAL;
1121	}
1122
1123	if (xts->key_len == AES_KEYSIZE_128)
1124		aestype = CCP_AES_TYPE_128;
1125	else if (xts->key_len == AES_KEYSIZE_256)
1126		aestype = CCP_AES_TYPE_256;
1127	else
1128		return -EINVAL;
1129
1130	if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1131		return -EINVAL;
1132
1133	if (xts->iv_len != AES_BLOCK_SIZE)
1134		return -EINVAL;
1135
1136	if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1137		return -EINVAL;
1138
1139	BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1140	BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1141
1142	ret = -EIO;
1143	memset(&op, 0, sizeof(op));
1144	op.cmd_q = cmd_q;
1145	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1146	op.sb_key = cmd_q->sb_key;
1147	op.sb_ctx = cmd_q->sb_ctx;
1148	op.init = 1;
1149	op.u.xts.type = aestype;
1150	op.u.xts.action = xts->action;
1151	op.u.xts.unit_size = xts->unit_size;
1152
1153	/* A version 3 device only supports 128-bit keys, which fits into a
1154	 * single SB entry. A version 5 device uses a 512-bit vector, so two
1155	 * SB entries.
1156	 */
1157	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1158		sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1159	else
1160		sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1161	ret = ccp_init_dm_workarea(&key, cmd_q,
1162				   sb_count * CCP_SB_BYTES,
1163				   DMA_TO_DEVICE);
1164	if (ret)
1165		return ret;
1166
1167	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1168		/* All supported key sizes must be in little endian format.
1169		 * Use the 256-bit byte swap passthru option to convert from
1170		 * big endian to little endian.
1171		 */
1172		dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1173		ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1174		if (ret)
1175			goto e_key;
1176		ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1177		if (ret)
1178			goto e_key;
1179	} else {
1180		/* Version 5 CCPs use a 512-bit space for the key: each portion
1181		 * occupies 256 bits, or one entire slot, and is zero-padded.
1182		 */
1183		unsigned int pad;
1184
1185		dm_offset = CCP_SB_BYTES;
1186		pad = dm_offset - xts->key_len;
1187		ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1188		if (ret)
1189			goto e_key;
1190		ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1191				      xts->key_len, xts->key_len);
1192		if (ret)
1193			goto e_key;
1194	}
1195	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1196			     CCP_PASSTHRU_BYTESWAP_256BIT);
1197	if (ret) {
1198		cmd->engine_error = cmd_q->cmd_error;
1199		goto e_key;
1200	}
1201
1202	/* The AES context fits in a single (32-byte) SB entry and
1203	 * for XTS is already in little endian format so no byte swapping
1204	 * is needed.
1205	 */
1206	ret = ccp_init_dm_workarea(&ctx, cmd_q,
1207				   CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1208				   DMA_BIDIRECTIONAL);
1209	if (ret)
1210		goto e_key;
1211
1212	ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1213	if (ret)
1214		goto e_ctx;
1215	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1216			     CCP_PASSTHRU_BYTESWAP_NOOP);
1217	if (ret) {
1218		cmd->engine_error = cmd_q->cmd_error;
1219		goto e_ctx;
1220	}
1221
1222	/* Prepare the input and output data workareas. For in-place
1223	 * operations we need to set the dma direction to BIDIRECTIONAL
1224	 * and copy the src workarea to the dst workarea.
1225	 */
1226	if (sg_virt(xts->src) == sg_virt(xts->dst))
1227		in_place = true;
1228
1229	ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1230			    unit_size,
1231			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1232	if (ret)
1233		goto e_ctx;
1234
1235	if (in_place) {
1236		dst = src;
1237	} else {
1238		ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1239				    unit_size, DMA_FROM_DEVICE);
1240		if (ret)
1241			goto e_src;
1242	}
1243
1244	/* Send data to the CCP AES engine */
1245	while (src.sg_wa.bytes_left) {
1246		ccp_prepare_data(&src, &dst, &op, unit_size, true);
1247		if (!src.sg_wa.bytes_left)
1248			op.eom = 1;
1249
1250		ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1251		if (ret) {
1252			cmd->engine_error = cmd_q->cmd_error;
1253			goto e_dst;
1254		}
1255
1256		ccp_process_data(&src, &dst, &op);
1257	}
1258
1259	/* Retrieve the AES context - convert from LE to BE using
1260	 * 32-byte (256-bit) byteswapping
1261	 */
1262	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1263			       CCP_PASSTHRU_BYTESWAP_256BIT);
1264	if (ret) {
1265		cmd->engine_error = cmd_q->cmd_error;
1266		goto e_dst;
1267	}
1268
1269	/* ...but we only need AES_BLOCK_SIZE bytes */
1270	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1271	ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1272
1273e_dst:
1274	if (!in_place)
1275		ccp_free_data(&dst, cmd_q);
1276
1277e_src:
1278	ccp_free_data(&src, cmd_q);
1279
1280e_ctx:
1281	ccp_dm_free(&ctx);
1282
1283e_key:
1284	ccp_dm_free(&key);
1285
1286	return ret;
1287}
1288
1289static noinline_for_stack int
1290ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1291{
1292	struct ccp_des3_engine *des3 = &cmd->u.des3;
1293
1294	struct ccp_dm_workarea key, ctx;
1295	struct ccp_data src, dst;
1296	struct ccp_op op;
1297	unsigned int dm_offset;
1298	unsigned int len_singlekey;
1299	bool in_place = false;
1300	int ret;
1301
1302	/* Error checks */
1303	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1304		return -EINVAL;
1305
1306	if (!cmd_q->ccp->vdata->perform->des3)
1307		return -EINVAL;
1308
1309	if (des3->key_len != DES3_EDE_KEY_SIZE)
1310		return -EINVAL;
1311
1312	if (((des3->mode == CCP_DES3_MODE_ECB) ||
1313		(des3->mode == CCP_DES3_MODE_CBC)) &&
1314		(des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1315		return -EINVAL;
1316
1317	if (!des3->key || !des3->src || !des3->dst)
1318		return -EINVAL;
1319
1320	if (des3->mode != CCP_DES3_MODE_ECB) {
1321		if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1322			return -EINVAL;
1323
1324		if (!des3->iv)
1325			return -EINVAL;
1326	}
1327
1328	/* Zero out all the fields of the command desc */
1329	memset(&op, 0, sizeof(op));
1330
1331	/* Set up the Function field */
1332	op.cmd_q = cmd_q;
1333	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1334	op.sb_key = cmd_q->sb_key;
1335
1336	op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1337	op.u.des3.type = des3->type;
1338	op.u.des3.mode = des3->mode;
1339	op.u.des3.action = des3->action;
1340
1341	/*
1342	 * All supported key sizes fit in a single (32-byte) KSB entry and
1343	 * (like AES) must be in little endian format. Use the 256-bit byte
1344	 * swap passthru option to convert from big endian to little endian.
1345	 */
1346	ret = ccp_init_dm_workarea(&key, cmd_q,
1347				   CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1348				   DMA_TO_DEVICE);
1349	if (ret)
1350		return ret;
1351
1352	/*
1353	 * The contents of the key triplet are in the reverse order of what
1354	 * is required by the engine. Copy the 3 pieces individually to put
1355	 * them where they belong.
1356	 */
1357	dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1358
1359	len_singlekey = des3->key_len / 3;
1360	ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1361			      des3->key, 0, len_singlekey);
1362	if (ret)
1363		goto e_key;
1364	ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1365			      des3->key, len_singlekey, len_singlekey);
1366	if (ret)
1367		goto e_key;
1368	ret = ccp_set_dm_area(&key, dm_offset,
1369			      des3->key, 2 * len_singlekey, len_singlekey);
1370	if (ret)
1371		goto e_key;
1372
1373	/* Copy the key to the SB */
1374	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1375			     CCP_PASSTHRU_BYTESWAP_256BIT);
1376	if (ret) {
1377		cmd->engine_error = cmd_q->cmd_error;
1378		goto e_key;
1379	}
1380
1381	/*
1382	 * The DES3 context fits in a single (32-byte) KSB entry and
1383	 * must be in little endian format. Use the 256-bit byte swap
1384	 * passthru option to convert from big endian to little endian.
1385	 */
1386	if (des3->mode != CCP_DES3_MODE_ECB) {
1387		op.sb_ctx = cmd_q->sb_ctx;
1388
1389		ret = ccp_init_dm_workarea(&ctx, cmd_q,
1390					   CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1391					   DMA_BIDIRECTIONAL);
1392		if (ret)
1393			goto e_key;
1394
1395		/* Load the context into the LSB */
1396		dm_offset = CCP_SB_BYTES - des3->iv_len;
1397		ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1398				      des3->iv_len);
1399		if (ret)
1400			goto e_ctx;
1401
1402		ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1403				     CCP_PASSTHRU_BYTESWAP_256BIT);
1404		if (ret) {
1405			cmd->engine_error = cmd_q->cmd_error;
1406			goto e_ctx;
1407		}
1408	}
1409
1410	/*
1411	 * Prepare the input and output data workareas. For in-place
1412	 * operations we need to set the dma direction to BIDIRECTIONAL
1413	 * and copy the src workarea to the dst workarea.
1414	 */
1415	if (sg_virt(des3->src) == sg_virt(des3->dst))
1416		in_place = true;
1417
1418	ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1419			DES3_EDE_BLOCK_SIZE,
1420			in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1421	if (ret)
1422		goto e_ctx;
1423
1424	if (in_place)
1425		dst = src;
1426	else {
1427		ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1428				DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1429		if (ret)
1430			goto e_src;
1431	}
1432
1433	/* Send data to the CCP DES3 engine */
1434	while (src.sg_wa.bytes_left) {
1435		ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1436		if (!src.sg_wa.bytes_left) {
1437			op.eom = 1;
1438
1439			/* Since we don't retrieve the context in ECB mode
1440			 * we have to wait for the operation to complete
1441			 * on the last piece of data
1442			 */
1443			op.soc = 0;
1444		}
1445
1446		ret = cmd_q->ccp->vdata->perform->des3(&op);
1447		if (ret) {
1448			cmd->engine_error = cmd_q->cmd_error;
1449			goto e_dst;
1450		}
1451
1452		ccp_process_data(&src, &dst, &op);
1453	}
1454
1455	if (des3->mode != CCP_DES3_MODE_ECB) {
1456		/* Retrieve the context and make BE */
1457		ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1458				       CCP_PASSTHRU_BYTESWAP_256BIT);
1459		if (ret) {
1460			cmd->engine_error = cmd_q->cmd_error;
1461			goto e_dst;
1462		}
1463
1464		/* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1465		ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1466				DES3_EDE_BLOCK_SIZE);
1467	}
1468e_dst:
1469	if (!in_place)
1470		ccp_free_data(&dst, cmd_q);
1471
1472e_src:
1473	ccp_free_data(&src, cmd_q);
1474
1475e_ctx:
1476	if (des3->mode != CCP_DES3_MODE_ECB)
1477		ccp_dm_free(&ctx);
1478
1479e_key:
1480	ccp_dm_free(&key);
1481
1482	return ret;
1483}
1484
1485static noinline_for_stack int
1486ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1487{
1488	struct ccp_sha_engine *sha = &cmd->u.sha;
1489	struct ccp_dm_workarea ctx;
1490	struct ccp_data src;
1491	struct ccp_op op;
1492	unsigned int ioffset, ooffset;
1493	unsigned int digest_size;
1494	int sb_count;
1495	const void *init;
1496	u64 block_size;
1497	int ctx_size;
1498	int ret;
1499
1500	switch (sha->type) {
1501	case CCP_SHA_TYPE_1:
1502		if (sha->ctx_len < SHA1_DIGEST_SIZE)
1503			return -EINVAL;
1504		block_size = SHA1_BLOCK_SIZE;
1505		break;
1506	case CCP_SHA_TYPE_224:
1507		if (sha->ctx_len < SHA224_DIGEST_SIZE)
1508			return -EINVAL;
1509		block_size = SHA224_BLOCK_SIZE;
1510		break;
1511	case CCP_SHA_TYPE_256:
1512		if (sha->ctx_len < SHA256_DIGEST_SIZE)
1513			return -EINVAL;
1514		block_size = SHA256_BLOCK_SIZE;
1515		break;
1516	case CCP_SHA_TYPE_384:
1517		if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1518		    || sha->ctx_len < SHA384_DIGEST_SIZE)
1519			return -EINVAL;
1520		block_size = SHA384_BLOCK_SIZE;
1521		break;
1522	case CCP_SHA_TYPE_512:
1523		if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1524		    || sha->ctx_len < SHA512_DIGEST_SIZE)
1525			return -EINVAL;
1526		block_size = SHA512_BLOCK_SIZE;
1527		break;
1528	default:
1529		return -EINVAL;
1530	}
1531
1532	if (!sha->ctx)
1533		return -EINVAL;
1534
1535	if (!sha->final && (sha->src_len & (block_size - 1)))
1536		return -EINVAL;
1537
1538	/* The version 3 device can't handle zero-length input */
1539	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1540
1541		if (!sha->src_len) {
1542			unsigned int digest_len;
1543			const u8 *sha_zero;
1544
1545			/* Not final, just return */
1546			if (!sha->final)
1547				return 0;
1548
1549			/* CCP can't do a zero length sha operation so the
1550			 * caller must buffer the data.
1551			 */
1552			if (sha->msg_bits)
1553				return -EINVAL;
1554
1555			/* The CCP cannot perform zero-length sha operations
1556			 * so the caller is required to buffer data for the
1557			 * final operation. However, a sha operation for a
1558			 * message with a total length of zero is valid so
1559			 * known values are required to supply the result.
1560			 */
1561			switch (sha->type) {
1562			case CCP_SHA_TYPE_1:
1563				sha_zero = sha1_zero_message_hash;
1564				digest_len = SHA1_DIGEST_SIZE;
1565				break;
1566			case CCP_SHA_TYPE_224:
1567				sha_zero = sha224_zero_message_hash;
1568				digest_len = SHA224_DIGEST_SIZE;
1569				break;
1570			case CCP_SHA_TYPE_256:
1571				sha_zero = sha256_zero_message_hash;
1572				digest_len = SHA256_DIGEST_SIZE;
1573				break;
1574			default:
1575				return -EINVAL;
1576			}
1577
1578			scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1579						 digest_len, 1);
1580
1581			return 0;
1582		}
1583	}
1584
1585	/* Set variables used throughout */
1586	switch (sha->type) {
1587	case CCP_SHA_TYPE_1:
1588		digest_size = SHA1_DIGEST_SIZE;
1589		init = (void *) ccp_sha1_init;
1590		ctx_size = SHA1_DIGEST_SIZE;
1591		sb_count = 1;
1592		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593			ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1594		else
1595			ooffset = ioffset = 0;
1596		break;
1597	case CCP_SHA_TYPE_224:
1598		digest_size = SHA224_DIGEST_SIZE;
1599		init = (void *) ccp_sha224_init;
1600		ctx_size = SHA256_DIGEST_SIZE;
1601		sb_count = 1;
1602		ioffset = 0;
1603		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1604			ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1605		else
1606			ooffset = 0;
1607		break;
1608	case CCP_SHA_TYPE_256:
1609		digest_size = SHA256_DIGEST_SIZE;
1610		init = (void *) ccp_sha256_init;
1611		ctx_size = SHA256_DIGEST_SIZE;
1612		sb_count = 1;
1613		ooffset = ioffset = 0;
1614		break;
1615	case CCP_SHA_TYPE_384:
1616		digest_size = SHA384_DIGEST_SIZE;
1617		init = (void *) ccp_sha384_init;
1618		ctx_size = SHA512_DIGEST_SIZE;
1619		sb_count = 2;
1620		ioffset = 0;
1621		ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1622		break;
1623	case CCP_SHA_TYPE_512:
1624		digest_size = SHA512_DIGEST_SIZE;
1625		init = (void *) ccp_sha512_init;
1626		ctx_size = SHA512_DIGEST_SIZE;
1627		sb_count = 2;
1628		ooffset = ioffset = 0;
1629		break;
1630	default:
1631		ret = -EINVAL;
1632		goto e_data;
1633	}
1634
1635	/* For zero-length plaintext the src pointer is ignored;
1636	 * otherwise both parts must be valid
1637	 */
1638	if (sha->src_len && !sha->src)
1639		return -EINVAL;
1640
1641	memset(&op, 0, sizeof(op));
1642	op.cmd_q = cmd_q;
1643	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1644	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1645	op.u.sha.type = sha->type;
1646	op.u.sha.msg_bits = sha->msg_bits;
1647
1648	/* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1649	 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1650	 * first slot, and the left half in the second. Each portion must then
1651	 * be in little endian format: use the 256-bit byte swap option.
1652	 */
1653	ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1654				   DMA_BIDIRECTIONAL);
1655	if (ret)
1656		return ret;
1657	if (sha->first) {
1658		switch (sha->type) {
1659		case CCP_SHA_TYPE_1:
1660		case CCP_SHA_TYPE_224:
1661		case CCP_SHA_TYPE_256:
1662			memcpy(ctx.address + ioffset, init, ctx_size);
1663			break;
1664		case CCP_SHA_TYPE_384:
1665		case CCP_SHA_TYPE_512:
1666			memcpy(ctx.address + ctx_size / 2, init,
1667			       ctx_size / 2);
1668			memcpy(ctx.address, init + ctx_size / 2,
1669			       ctx_size / 2);
1670			break;
1671		default:
1672			ret = -EINVAL;
1673			goto e_ctx;
1674		}
1675	} else {
1676		/* Restore the context */
1677		ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1678				      sb_count * CCP_SB_BYTES);
1679		if (ret)
1680			goto e_ctx;
1681	}
1682
1683	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1684			     CCP_PASSTHRU_BYTESWAP_256BIT);
1685	if (ret) {
1686		cmd->engine_error = cmd_q->cmd_error;
1687		goto e_ctx;
1688	}
1689
1690	if (sha->src) {
1691		/* Send data to the CCP SHA engine; block_size is set above */
1692		ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1693				    block_size, DMA_TO_DEVICE);
1694		if (ret)
1695			goto e_ctx;
1696
1697		while (src.sg_wa.bytes_left) {
1698			ccp_prepare_data(&src, NULL, &op, block_size, false);
1699			if (sha->final && !src.sg_wa.bytes_left)
1700				op.eom = 1;
1701
1702			ret = cmd_q->ccp->vdata->perform->sha(&op);
1703			if (ret) {
1704				cmd->engine_error = cmd_q->cmd_error;
1705				goto e_data;
1706			}
1707
1708			ccp_process_data(&src, NULL, &op);
1709		}
1710	} else {
1711		op.eom = 1;
1712		ret = cmd_q->ccp->vdata->perform->sha(&op);
1713		if (ret) {
1714			cmd->engine_error = cmd_q->cmd_error;
1715			goto e_data;
1716		}
1717	}
1718
1719	/* Retrieve the SHA context - convert from LE to BE using
1720	 * 32-byte (256-bit) byteswapping to BE
1721	 */
1722	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1723			       CCP_PASSTHRU_BYTESWAP_256BIT);
1724	if (ret) {
1725		cmd->engine_error = cmd_q->cmd_error;
1726		goto e_data;
1727	}
1728
1729	if (sha->final) {
1730		/* Finishing up, so get the digest */
1731		switch (sha->type) {
1732		case CCP_SHA_TYPE_1:
1733		case CCP_SHA_TYPE_224:
1734		case CCP_SHA_TYPE_256:
1735			ccp_get_dm_area(&ctx, ooffset,
1736					sha->ctx, 0,
1737					digest_size);
1738			break;
1739		case CCP_SHA_TYPE_384:
1740		case CCP_SHA_TYPE_512:
1741			ccp_get_dm_area(&ctx, 0,
1742					sha->ctx, LSB_ITEM_SIZE - ooffset,
1743					LSB_ITEM_SIZE);
1744			ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1745					sha->ctx, 0,
1746					LSB_ITEM_SIZE - ooffset);
1747			break;
1748		default:
1749			ret = -EINVAL;
1750			goto e_data;
1751		}
1752	} else {
1753		/* Stash the context */
1754		ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1755				sb_count * CCP_SB_BYTES);
1756	}
1757
1758	if (sha->final && sha->opad) {
1759		/* HMAC operation, recursively perform final SHA */
1760		struct ccp_cmd hmac_cmd;
1761		struct scatterlist sg;
1762		u8 *hmac_buf;
1763
1764		if (sha->opad_len != block_size) {
1765			ret = -EINVAL;
1766			goto e_data;
1767		}
1768
1769		hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1770		if (!hmac_buf) {
1771			ret = -ENOMEM;
1772			goto e_data;
1773		}
1774		sg_init_one(&sg, hmac_buf, block_size + digest_size);
1775
1776		scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1777		switch (sha->type) {
1778		case CCP_SHA_TYPE_1:
1779		case CCP_SHA_TYPE_224:
1780		case CCP_SHA_TYPE_256:
1781			memcpy(hmac_buf + block_size,
1782			       ctx.address + ooffset,
1783			       digest_size);
1784			break;
1785		case CCP_SHA_TYPE_384:
1786		case CCP_SHA_TYPE_512:
1787			memcpy(hmac_buf + block_size,
1788			       ctx.address + LSB_ITEM_SIZE + ooffset,
1789			       LSB_ITEM_SIZE);
1790			memcpy(hmac_buf + block_size +
1791			       (LSB_ITEM_SIZE - ooffset),
1792			       ctx.address,
1793			       LSB_ITEM_SIZE);
1794			break;
1795		default:
1796			kfree(hmac_buf);
1797			ret = -EINVAL;
1798			goto e_data;
1799		}
1800
1801		memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1802		hmac_cmd.engine = CCP_ENGINE_SHA;
1803		hmac_cmd.u.sha.type = sha->type;
1804		hmac_cmd.u.sha.ctx = sha->ctx;
1805		hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1806		hmac_cmd.u.sha.src = &sg;
1807		hmac_cmd.u.sha.src_len = block_size + digest_size;
1808		hmac_cmd.u.sha.opad = NULL;
1809		hmac_cmd.u.sha.opad_len = 0;
1810		hmac_cmd.u.sha.first = 1;
1811		hmac_cmd.u.sha.final = 1;
1812		hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1813
1814		ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1815		if (ret)
1816			cmd->engine_error = hmac_cmd.engine_error;
1817
1818		kfree(hmac_buf);
1819	}
1820
1821e_data:
1822	if (sha->src)
1823		ccp_free_data(&src, cmd_q);
1824
1825e_ctx:
1826	ccp_dm_free(&ctx);
1827
1828	return ret;
1829}
1830
1831static noinline_for_stack int
1832ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1833{
1834	struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1835	struct ccp_dm_workarea exp, src, dst;
1836	struct ccp_op op;
1837	unsigned int sb_count, i_len, o_len;
1838	int ret;
1839
1840	/* Check against the maximum allowable size, in bits */
1841	if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1842		return -EINVAL;
1843
1844	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1845		return -EINVAL;
1846
1847	memset(&op, 0, sizeof(op));
1848	op.cmd_q = cmd_q;
1849	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1850
1851	/* The RSA modulus must precede the message being acted upon, so
1852	 * it must be copied to a DMA area where the message and the
1853	 * modulus can be concatenated.  Therefore the input buffer
1854	 * length required is twice the output buffer length (which
1855	 * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1856	 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1857	 * required.
1858	 */
1859	o_len = 32 * ((rsa->key_size + 255) / 256);
1860	i_len = o_len * 2;
1861
1862	sb_count = 0;
1863	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1864		/* sb_count is the number of storage block slots required
1865		 * for the modulus.
1866		 */
1867		sb_count = o_len / CCP_SB_BYTES;
1868		op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1869								sb_count);
1870		if (!op.sb_key)
1871			return -EIO;
1872	} else {
1873		/* A version 5 device allows a modulus size that will not fit
1874		 * in the LSB, so the command will transfer it from memory.
1875		 * Set the sb key to the default, even though it's not used.
1876		 */
1877		op.sb_key = cmd_q->sb_key;
1878	}
1879
1880	/* The RSA exponent must be in little endian format. Reverse its
1881	 * byte order.
1882	 */
1883	ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1884	if (ret)
1885		goto e_sb;
1886
1887	ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1888	if (ret)
1889		goto e_exp;
1890
1891	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1892		/* Copy the exponent to the local storage block, using
1893		 * as many 32-byte blocks as were allocated above. It's
1894		 * already little endian, so no further change is required.
1895		 */
1896		ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1897				     CCP_PASSTHRU_BYTESWAP_NOOP);
1898		if (ret) {
1899			cmd->engine_error = cmd_q->cmd_error;
1900			goto e_exp;
1901		}
1902	} else {
1903		/* The exponent can be retrieved from memory via DMA. */
1904		op.exp.u.dma.address = exp.dma.address;
1905		op.exp.u.dma.offset = 0;
1906	}
1907
1908	/* Concatenate the modulus and the message. Both the modulus and
1909	 * the operands must be in little endian format.  Since the input
1910	 * is in big endian format it must be converted.
1911	 */
1912	ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1913	if (ret)
1914		goto e_exp;
1915
1916	ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1917	if (ret)
1918		goto e_src;
1919	ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1920	if (ret)
1921		goto e_src;
1922
1923	/* Prepare the output area for the operation */
1924	ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1925	if (ret)
1926		goto e_src;
1927
1928	op.soc = 1;
1929	op.src.u.dma.address = src.dma.address;
1930	op.src.u.dma.offset = 0;
1931	op.src.u.dma.length = i_len;
1932	op.dst.u.dma.address = dst.dma.address;
1933	op.dst.u.dma.offset = 0;
1934	op.dst.u.dma.length = o_len;
1935
1936	op.u.rsa.mod_size = rsa->key_size;
1937	op.u.rsa.input_len = i_len;
1938
1939	ret = cmd_q->ccp->vdata->perform->rsa(&op);
1940	if (ret) {
1941		cmd->engine_error = cmd_q->cmd_error;
1942		goto e_dst;
1943	}
1944
1945	ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1946
1947e_dst:
1948	ccp_dm_free(&dst);
1949
1950e_src:
1951	ccp_dm_free(&src);
1952
1953e_exp:
1954	ccp_dm_free(&exp);
1955
1956e_sb:
1957	if (sb_count)
1958		cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1959
1960	return ret;
1961}
1962
1963static noinline_for_stack int
1964ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1965{
1966	struct ccp_passthru_engine *pt = &cmd->u.passthru;
1967	struct ccp_dm_workarea mask;
1968	struct ccp_data src, dst;
1969	struct ccp_op op;
1970	bool in_place = false;
1971	unsigned int i;
1972	int ret = 0;
1973
1974	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1975		return -EINVAL;
1976
1977	if (!pt->src || !pt->dst)
1978		return -EINVAL;
1979
1980	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981		if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1982			return -EINVAL;
1983		if (!pt->mask)
1984			return -EINVAL;
1985	}
1986
1987	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1988
1989	memset(&op, 0, sizeof(op));
1990	op.cmd_q = cmd_q;
1991	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1992
1993	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1994		/* Load the mask */
1995		op.sb_key = cmd_q->sb_key;
1996
1997		ret = ccp_init_dm_workarea(&mask, cmd_q,
1998					   CCP_PASSTHRU_SB_COUNT *
1999					   CCP_SB_BYTES,
2000					   DMA_TO_DEVICE);
2001		if (ret)
2002			return ret;
2003
2004		ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2005		if (ret)
2006			goto e_mask;
2007		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2008				     CCP_PASSTHRU_BYTESWAP_NOOP);
2009		if (ret) {
2010			cmd->engine_error = cmd_q->cmd_error;
2011			goto e_mask;
2012		}
2013	}
2014
2015	/* Prepare the input and output data workareas. For in-place
2016	 * operations we need to set the dma direction to BIDIRECTIONAL
2017	 * and copy the src workarea to the dst workarea.
2018	 */
2019	if (sg_virt(pt->src) == sg_virt(pt->dst))
2020		in_place = true;
2021
2022	ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2023			    CCP_PASSTHRU_MASKSIZE,
2024			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2025	if (ret)
2026		goto e_mask;
2027
2028	if (in_place) {
2029		dst = src;
2030	} else {
2031		ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2032				    CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2033		if (ret)
2034			goto e_src;
2035	}
2036
2037	/* Send data to the CCP Passthru engine
2038	 *   Because the CCP engine works on a single source and destination
2039	 *   dma address at a time, each entry in the source scatterlist
2040	 *   (after the dma_map_sg call) must be less than or equal to the
2041	 *   (remaining) length in the destination scatterlist entry and the
2042	 *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2043	 */
2044	dst.sg_wa.sg_used = 0;
2045	for (i = 1; i <= src.sg_wa.dma_count; i++) {
2046		if (!dst.sg_wa.sg ||
2047		    (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2048			ret = -EINVAL;
2049			goto e_dst;
2050		}
2051
2052		if (i == src.sg_wa.dma_count) {
2053			op.eom = 1;
2054			op.soc = 1;
2055		}
2056
2057		op.src.type = CCP_MEMTYPE_SYSTEM;
2058		op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2059		op.src.u.dma.offset = 0;
2060		op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2061
2062		op.dst.type = CCP_MEMTYPE_SYSTEM;
2063		op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2064		op.dst.u.dma.offset = dst.sg_wa.sg_used;
2065		op.dst.u.dma.length = op.src.u.dma.length;
2066
2067		ret = cmd_q->ccp->vdata->perform->passthru(&op);
2068		if (ret) {
2069			cmd->engine_error = cmd_q->cmd_error;
2070			goto e_dst;
2071		}
2072
2073		dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2074		if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2075			dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2076			dst.sg_wa.sg_used = 0;
2077		}
2078		src.sg_wa.sg = sg_next(src.sg_wa.sg);
2079	}
2080
2081e_dst:
2082	if (!in_place)
2083		ccp_free_data(&dst, cmd_q);
2084
2085e_src:
2086	ccp_free_data(&src, cmd_q);
2087
2088e_mask:
2089	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2090		ccp_dm_free(&mask);
2091
2092	return ret;
2093}
2094
2095static noinline_for_stack int
2096ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2097				      struct ccp_cmd *cmd)
2098{
2099	struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2100	struct ccp_dm_workarea mask;
2101	struct ccp_op op;
2102	int ret;
2103
2104	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2105		return -EINVAL;
2106
2107	if (!pt->src_dma || !pt->dst_dma)
2108		return -EINVAL;
2109
2110	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2111		if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2112			return -EINVAL;
2113		if (!pt->mask)
2114			return -EINVAL;
2115	}
2116
2117	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2118
2119	memset(&op, 0, sizeof(op));
2120	op.cmd_q = cmd_q;
2121	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2122
2123	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2124		/* Load the mask */
2125		op.sb_key = cmd_q->sb_key;
2126
2127		mask.length = pt->mask_len;
2128		mask.dma.address = pt->mask;
2129		mask.dma.length = pt->mask_len;
2130
2131		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2132				     CCP_PASSTHRU_BYTESWAP_NOOP);
2133		if (ret) {
2134			cmd->engine_error = cmd_q->cmd_error;
2135			return ret;
2136		}
2137	}
2138
2139	/* Send data to the CCP Passthru engine */
2140	op.eom = 1;
2141	op.soc = 1;
2142
2143	op.src.type = CCP_MEMTYPE_SYSTEM;
2144	op.src.u.dma.address = pt->src_dma;
2145	op.src.u.dma.offset = 0;
2146	op.src.u.dma.length = pt->src_len;
2147
2148	op.dst.type = CCP_MEMTYPE_SYSTEM;
2149	op.dst.u.dma.address = pt->dst_dma;
2150	op.dst.u.dma.offset = 0;
2151	op.dst.u.dma.length = pt->src_len;
2152
2153	ret = cmd_q->ccp->vdata->perform->passthru(&op);
2154	if (ret)
2155		cmd->engine_error = cmd_q->cmd_error;
2156
2157	return ret;
2158}
2159
2160static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2161{
2162	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2163	struct ccp_dm_workarea src, dst;
2164	struct ccp_op op;
2165	int ret;
2166	u8 *save;
2167
2168	if (!ecc->u.mm.operand_1 ||
2169	    (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2170		return -EINVAL;
2171
2172	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2173		if (!ecc->u.mm.operand_2 ||
2174		    (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2175			return -EINVAL;
2176
2177	if (!ecc->u.mm.result ||
2178	    (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2179		return -EINVAL;
2180
2181	memset(&op, 0, sizeof(op));
2182	op.cmd_q = cmd_q;
2183	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2184
2185	/* Concatenate the modulus and the operands. Both the modulus and
2186	 * the operands must be in little endian format.  Since the input
2187	 * is in big endian format it must be converted and placed in a
2188	 * fixed length buffer.
2189	 */
2190	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2191				   DMA_TO_DEVICE);
2192	if (ret)
2193		return ret;
2194
2195	/* Save the workarea address since it is updated in order to perform
2196	 * the concatenation
2197	 */
2198	save = src.address;
2199
2200	/* Copy the ECC modulus */
2201	ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2202	if (ret)
2203		goto e_src;
2204	src.address += CCP_ECC_OPERAND_SIZE;
2205
2206	/* Copy the first operand */
2207	ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2208				      ecc->u.mm.operand_1_len);
2209	if (ret)
2210		goto e_src;
2211	src.address += CCP_ECC_OPERAND_SIZE;
2212
2213	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2214		/* Copy the second operand */
2215		ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2216					      ecc->u.mm.operand_2_len);
2217		if (ret)
2218			goto e_src;
2219		src.address += CCP_ECC_OPERAND_SIZE;
2220	}
2221
2222	/* Restore the workarea address */
2223	src.address = save;
2224
2225	/* Prepare the output area for the operation */
2226	ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2227				   DMA_FROM_DEVICE);
2228	if (ret)
2229		goto e_src;
2230
2231	op.soc = 1;
2232	op.src.u.dma.address = src.dma.address;
2233	op.src.u.dma.offset = 0;
2234	op.src.u.dma.length = src.length;
2235	op.dst.u.dma.address = dst.dma.address;
2236	op.dst.u.dma.offset = 0;
2237	op.dst.u.dma.length = dst.length;
2238
2239	op.u.ecc.function = cmd->u.ecc.function;
2240
2241	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2242	if (ret) {
2243		cmd->engine_error = cmd_q->cmd_error;
2244		goto e_dst;
2245	}
2246
2247	ecc->ecc_result = le16_to_cpup(
2248		(const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2249	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2250		ret = -EIO;
2251		goto e_dst;
2252	}
2253
2254	/* Save the ECC result */
2255	ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2256				CCP_ECC_MODULUS_BYTES);
2257
2258e_dst:
2259	ccp_dm_free(&dst);
2260
2261e_src:
2262	ccp_dm_free(&src);
2263
2264	return ret;
2265}
2266
2267static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2268{
2269	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2270	struct ccp_dm_workarea src, dst;
2271	struct ccp_op op;
2272	int ret;
2273	u8 *save;
2274
2275	if (!ecc->u.pm.point_1.x ||
2276	    (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2277	    !ecc->u.pm.point_1.y ||
2278	    (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2279		return -EINVAL;
2280
2281	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2282		if (!ecc->u.pm.point_2.x ||
2283		    (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2284		    !ecc->u.pm.point_2.y ||
2285		    (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2286			return -EINVAL;
2287	} else {
2288		if (!ecc->u.pm.domain_a ||
2289		    (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2290			return -EINVAL;
2291
2292		if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2293			if (!ecc->u.pm.scalar ||
2294			    (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2295				return -EINVAL;
2296	}
2297
2298	if (!ecc->u.pm.result.x ||
2299	    (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2300	    !ecc->u.pm.result.y ||
2301	    (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2302		return -EINVAL;
2303
2304	memset(&op, 0, sizeof(op));
2305	op.cmd_q = cmd_q;
2306	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2307
2308	/* Concatenate the modulus and the operands. Both the modulus and
2309	 * the operands must be in little endian format.  Since the input
2310	 * is in big endian format it must be converted and placed in a
2311	 * fixed length buffer.
2312	 */
2313	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2314				   DMA_TO_DEVICE);
2315	if (ret)
2316		return ret;
2317
2318	/* Save the workarea address since it is updated in order to perform
2319	 * the concatenation
2320	 */
2321	save = src.address;
2322
2323	/* Copy the ECC modulus */
2324	ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2325	if (ret)
2326		goto e_src;
2327	src.address += CCP_ECC_OPERAND_SIZE;
2328
2329	/* Copy the first point X and Y coordinate */
2330	ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2331				      ecc->u.pm.point_1.x_len);
2332	if (ret)
2333		goto e_src;
2334	src.address += CCP_ECC_OPERAND_SIZE;
2335	ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2336				      ecc->u.pm.point_1.y_len);
2337	if (ret)
2338		goto e_src;
2339	src.address += CCP_ECC_OPERAND_SIZE;
2340
2341	/* Set the first point Z coordinate to 1 */
2342	*src.address = 0x01;
2343	src.address += CCP_ECC_OPERAND_SIZE;
2344
2345	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2346		/* Copy the second point X and Y coordinate */
2347		ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2348					      ecc->u.pm.point_2.x_len);
2349		if (ret)
2350			goto e_src;
2351		src.address += CCP_ECC_OPERAND_SIZE;
2352		ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2353					      ecc->u.pm.point_2.y_len);
2354		if (ret)
2355			goto e_src;
2356		src.address += CCP_ECC_OPERAND_SIZE;
2357
2358		/* Set the second point Z coordinate to 1 */
2359		*src.address = 0x01;
2360		src.address += CCP_ECC_OPERAND_SIZE;
2361	} else {
2362		/* Copy the Domain "a" parameter */
2363		ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2364					      ecc->u.pm.domain_a_len);
2365		if (ret)
2366			goto e_src;
2367		src.address += CCP_ECC_OPERAND_SIZE;
2368
2369		if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2370			/* Copy the scalar value */
2371			ret = ccp_reverse_set_dm_area(&src, 0,
2372						      ecc->u.pm.scalar, 0,
2373						      ecc->u.pm.scalar_len);
2374			if (ret)
2375				goto e_src;
2376			src.address += CCP_ECC_OPERAND_SIZE;
2377		}
2378	}
2379
2380	/* Restore the workarea address */
2381	src.address = save;
2382
2383	/* Prepare the output area for the operation */
2384	ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2385				   DMA_FROM_DEVICE);
2386	if (ret)
2387		goto e_src;
2388
2389	op.soc = 1;
2390	op.src.u.dma.address = src.dma.address;
2391	op.src.u.dma.offset = 0;
2392	op.src.u.dma.length = src.length;
2393	op.dst.u.dma.address = dst.dma.address;
2394	op.dst.u.dma.offset = 0;
2395	op.dst.u.dma.length = dst.length;
2396
2397	op.u.ecc.function = cmd->u.ecc.function;
2398
2399	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2400	if (ret) {
2401		cmd->engine_error = cmd_q->cmd_error;
2402		goto e_dst;
2403	}
2404
2405	ecc->ecc_result = le16_to_cpup(
2406		(const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2407	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2408		ret = -EIO;
2409		goto e_dst;
2410	}
2411
2412	/* Save the workarea address since it is updated as we walk through
2413	 * to copy the point math result
2414	 */
2415	save = dst.address;
2416
2417	/* Save the ECC result X and Y coordinates */
2418	ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2419				CCP_ECC_MODULUS_BYTES);
2420	dst.address += CCP_ECC_OUTPUT_SIZE;
2421	ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2422				CCP_ECC_MODULUS_BYTES);
 
2423
2424	/* Restore the workarea address */
2425	dst.address = save;
2426
2427e_dst:
2428	ccp_dm_free(&dst);
2429
2430e_src:
2431	ccp_dm_free(&src);
2432
2433	return ret;
2434}
2435
2436static noinline_for_stack int
2437ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2438{
2439	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2440
2441	ecc->ecc_result = 0;
2442
2443	if (!ecc->mod ||
2444	    (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2445		return -EINVAL;
2446
2447	switch (ecc->function) {
2448	case CCP_ECC_FUNCTION_MMUL_384BIT:
2449	case CCP_ECC_FUNCTION_MADD_384BIT:
2450	case CCP_ECC_FUNCTION_MINV_384BIT:
2451		return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2452
2453	case CCP_ECC_FUNCTION_PADD_384BIT:
2454	case CCP_ECC_FUNCTION_PMUL_384BIT:
2455	case CCP_ECC_FUNCTION_PDBL_384BIT:
2456		return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2457
2458	default:
2459		return -EINVAL;
2460	}
2461}
2462
2463int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2464{
2465	int ret;
2466
2467	cmd->engine_error = 0;
2468	cmd_q->cmd_error = 0;
2469	cmd_q->int_rcvd = 0;
2470	cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2471
2472	switch (cmd->engine) {
2473	case CCP_ENGINE_AES:
2474		switch (cmd->u.aes.mode) {
2475		case CCP_AES_MODE_CMAC:
2476			ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2477			break;
2478		case CCP_AES_MODE_GCM:
2479			ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2480			break;
2481		default:
2482			ret = ccp_run_aes_cmd(cmd_q, cmd);
2483			break;
2484		}
2485		break;
2486	case CCP_ENGINE_XTS_AES_128:
2487		ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2488		break;
2489	case CCP_ENGINE_DES3:
2490		ret = ccp_run_des3_cmd(cmd_q, cmd);
2491		break;
2492	case CCP_ENGINE_SHA:
2493		ret = ccp_run_sha_cmd(cmd_q, cmd);
2494		break;
2495	case CCP_ENGINE_RSA:
2496		ret = ccp_run_rsa_cmd(cmd_q, cmd);
2497		break;
2498	case CCP_ENGINE_PASSTHRU:
2499		if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2500			ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2501		else
2502			ret = ccp_run_passthru_cmd(cmd_q, cmd);
2503		break;
2504	case CCP_ENGINE_ECC:
2505		ret = ccp_run_ecc_cmd(cmd_q, cmd);
2506		break;
2507	default:
2508		ret = -EINVAL;
2509	}
2510
2511	return ret;
2512}