1// SPDX-License-Identifier: GPL-2.0
   2//
   3// Cryptographic API.
   4//
   5// Support for Samsung S5PV210 and Exynos HW acceleration.
   6//
   7// Copyright (C) 2011 NetUP Inc. All rights reserved.
   8// Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.
   9//
  10// Hash part based on omap-sham.c driver.
  11
  12#include <linux/clk.h>
  13#include <linux/crypto.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/err.h>
  16#include <linux/errno.h>
  17#include <linux/init.h>
  18#include <linux/interrupt.h>
  19#include <linux/io.h>
  20#include <linux/kernel.h>
  21#include <linux/module.h>
  22#include <linux/of.h>
 
  23#include <linux/platform_device.h>
  24#include <linux/scatterlist.h>
  25
  26#include <crypto/ctr.h>
  27#include <crypto/aes.h>
  28#include <crypto/algapi.h>
  29#include <crypto/scatterwalk.h>
  30
  31#include <crypto/hash.h>
  32#include <crypto/md5.h>
  33#include <crypto/sha1.h>
  34#include <crypto/sha2.h>
  35#include <crypto/internal/hash.h>
  36
  37#define _SBF(s, v)			((v) << (s))
  38
  39/* Feed control registers */
  40#define SSS_REG_FCINTSTAT		0x0000
  41#define SSS_FCINTSTAT_HPARTINT		BIT(7)
  42#define SSS_FCINTSTAT_HDONEINT		BIT(5)
  43#define SSS_FCINTSTAT_BRDMAINT		BIT(3)
  44#define SSS_FCINTSTAT_BTDMAINT		BIT(2)
  45#define SSS_FCINTSTAT_HRDMAINT		BIT(1)
  46#define SSS_FCINTSTAT_PKDMAINT		BIT(0)
  47
  48#define SSS_REG_FCINTENSET		0x0004
  49#define SSS_FCINTENSET_HPARTINTENSET	BIT(7)
  50#define SSS_FCINTENSET_HDONEINTENSET	BIT(5)
  51#define SSS_FCINTENSET_BRDMAINTENSET	BIT(3)
  52#define SSS_FCINTENSET_BTDMAINTENSET	BIT(2)
  53#define SSS_FCINTENSET_HRDMAINTENSET	BIT(1)
  54#define SSS_FCINTENSET_PKDMAINTENSET	BIT(0)
  55
  56#define SSS_REG_FCINTENCLR		0x0008
  57#define SSS_FCINTENCLR_HPARTINTENCLR	BIT(7)
  58#define SSS_FCINTENCLR_HDONEINTENCLR	BIT(5)
  59#define SSS_FCINTENCLR_BRDMAINTENCLR	BIT(3)
  60#define SSS_FCINTENCLR_BTDMAINTENCLR	BIT(2)
  61#define SSS_FCINTENCLR_HRDMAINTENCLR	BIT(1)
  62#define SSS_FCINTENCLR_PKDMAINTENCLR	BIT(0)
  63
  64#define SSS_REG_FCINTPEND		0x000C
  65#define SSS_FCINTPEND_HPARTINTP		BIT(7)
  66#define SSS_FCINTPEND_HDONEINTP		BIT(5)
  67#define SSS_FCINTPEND_BRDMAINTP		BIT(3)
  68#define SSS_FCINTPEND_BTDMAINTP		BIT(2)
  69#define SSS_FCINTPEND_HRDMAINTP		BIT(1)
  70#define SSS_FCINTPEND_PKDMAINTP		BIT(0)
  71
  72#define SSS_REG_FCFIFOSTAT		0x0010
  73#define SSS_FCFIFOSTAT_BRFIFOFUL	BIT(7)
  74#define SSS_FCFIFOSTAT_BRFIFOEMP	BIT(6)
  75#define SSS_FCFIFOSTAT_BTFIFOFUL	BIT(5)
  76#define SSS_FCFIFOSTAT_BTFIFOEMP	BIT(4)
  77#define SSS_FCFIFOSTAT_HRFIFOFUL	BIT(3)
  78#define SSS_FCFIFOSTAT_HRFIFOEMP	BIT(2)
  79#define SSS_FCFIFOSTAT_PKFIFOFUL	BIT(1)
  80#define SSS_FCFIFOSTAT_PKFIFOEMP	BIT(0)
  81
  82#define SSS_REG_FCFIFOCTRL		0x0014
  83#define SSS_FCFIFOCTRL_DESSEL		BIT(2)
  84#define SSS_HASHIN_INDEPENDENT		_SBF(0, 0x00)
  85#define SSS_HASHIN_CIPHER_INPUT		_SBF(0, 0x01)
  86#define SSS_HASHIN_CIPHER_OUTPUT	_SBF(0, 0x02)
  87#define SSS_HASHIN_MASK			_SBF(0, 0x03)
  88
  89#define SSS_REG_FCBRDMAS		0x0020
  90#define SSS_REG_FCBRDMAL		0x0024
  91#define SSS_REG_FCBRDMAC		0x0028
  92#define SSS_FCBRDMAC_BYTESWAP		BIT(1)
  93#define SSS_FCBRDMAC_FLUSH		BIT(0)
  94
  95#define SSS_REG_FCBTDMAS		0x0030
  96#define SSS_REG_FCBTDMAL		0x0034
  97#define SSS_REG_FCBTDMAC		0x0038
  98#define SSS_FCBTDMAC_BYTESWAP		BIT(1)
  99#define SSS_FCBTDMAC_FLUSH		BIT(0)
 100
 101#define SSS_REG_FCHRDMAS		0x0040
 102#define SSS_REG_FCHRDMAL		0x0044
 103#define SSS_REG_FCHRDMAC		0x0048
 104#define SSS_FCHRDMAC_BYTESWAP		BIT(1)
 105#define SSS_FCHRDMAC_FLUSH		BIT(0)
 106
 107#define SSS_REG_FCPKDMAS		0x0050
 108#define SSS_REG_FCPKDMAL		0x0054
 109#define SSS_REG_FCPKDMAC		0x0058
 110#define SSS_FCPKDMAC_BYTESWAP		BIT(3)
 111#define SSS_FCPKDMAC_DESCEND		BIT(2)
 112#define SSS_FCPKDMAC_TRANSMIT		BIT(1)
 113#define SSS_FCPKDMAC_FLUSH		BIT(0)
 114
 115#define SSS_REG_FCPKDMAO		0x005C
 116
 117/* AES registers */
 118#define SSS_REG_AES_CONTROL		0x00
 119#define SSS_AES_BYTESWAP_DI		BIT(11)
 120#define SSS_AES_BYTESWAP_DO		BIT(10)
 121#define SSS_AES_BYTESWAP_IV		BIT(9)
 122#define SSS_AES_BYTESWAP_CNT		BIT(8)
 123#define SSS_AES_BYTESWAP_KEY		BIT(7)
 124#define SSS_AES_KEY_CHANGE_MODE		BIT(6)
 125#define SSS_AES_KEY_SIZE_128		_SBF(4, 0x00)
 126#define SSS_AES_KEY_SIZE_192		_SBF(4, 0x01)
 127#define SSS_AES_KEY_SIZE_256		_SBF(4, 0x02)
 128#define SSS_AES_FIFO_MODE		BIT(3)
 129#define SSS_AES_CHAIN_MODE_ECB		_SBF(1, 0x00)
 130#define SSS_AES_CHAIN_MODE_CBC		_SBF(1, 0x01)
 131#define SSS_AES_CHAIN_MODE_CTR		_SBF(1, 0x02)
 132#define SSS_AES_MODE_DECRYPT		BIT(0)
 133
 134#define SSS_REG_AES_STATUS		0x04
 135#define SSS_AES_BUSY			BIT(2)
 136#define SSS_AES_INPUT_READY		BIT(1)
 137#define SSS_AES_OUTPUT_READY		BIT(0)
 138
 139#define SSS_REG_AES_IN_DATA(s)		(0x10 + (s << 2))
 140#define SSS_REG_AES_OUT_DATA(s)		(0x20 + (s << 2))
 141#define SSS_REG_AES_IV_DATA(s)		(0x30 + (s << 2))
 142#define SSS_REG_AES_CNT_DATA(s)		(0x40 + (s << 2))
 143#define SSS_REG_AES_KEY_DATA(s)		(0x80 + (s << 2))
 144
 145#define SSS_REG(dev, reg)		((dev)->ioaddr + (SSS_REG_##reg))
 146#define SSS_READ(dev, reg)		__raw_readl(SSS_REG(dev, reg))
 147#define SSS_WRITE(dev, reg, val)	__raw_writel((val), SSS_REG(dev, reg))
 148
 149#define SSS_AES_REG(dev, reg)		((dev)->aes_ioaddr + SSS_REG_##reg)
 150#define SSS_AES_WRITE(dev, reg, val)    __raw_writel((val), \
 151						SSS_AES_REG(dev, reg))
 152
 153/* HW engine modes */
 154#define FLAGS_AES_DECRYPT		BIT(0)
 155#define FLAGS_AES_MODE_MASK		_SBF(1, 0x03)
 156#define FLAGS_AES_CBC			_SBF(1, 0x01)
 157#define FLAGS_AES_CTR			_SBF(1, 0x02)
 158
 159#define AES_KEY_LEN			16
 160#define CRYPTO_QUEUE_LEN		1
 161
 162/* HASH registers */
 163#define SSS_REG_HASH_CTRL		0x00
 164
 165#define SSS_HASH_USER_IV_EN		BIT(5)
 166#define SSS_HASH_INIT_BIT		BIT(4)
 167#define SSS_HASH_ENGINE_SHA1		_SBF(1, 0x00)
 168#define SSS_HASH_ENGINE_MD5		_SBF(1, 0x01)
 169#define SSS_HASH_ENGINE_SHA256		_SBF(1, 0x02)
 170
 171#define SSS_HASH_ENGINE_MASK		_SBF(1, 0x03)
 172
 173#define SSS_REG_HASH_CTRL_PAUSE		0x04
 174
 175#define SSS_HASH_PAUSE			BIT(0)
 176
 177#define SSS_REG_HASH_CTRL_FIFO		0x08
 178
 179#define SSS_HASH_FIFO_MODE_DMA		BIT(0)
 180#define SSS_HASH_FIFO_MODE_CPU          0
 181
 182#define SSS_REG_HASH_CTRL_SWAP		0x0C
 183
 184#define SSS_HASH_BYTESWAP_DI		BIT(3)
 185#define SSS_HASH_BYTESWAP_DO		BIT(2)
 186#define SSS_HASH_BYTESWAP_IV		BIT(1)
 187#define SSS_HASH_BYTESWAP_KEY		BIT(0)
 188
 189#define SSS_REG_HASH_STATUS		0x10
 190
 191#define SSS_HASH_STATUS_MSG_DONE	BIT(6)
 192#define SSS_HASH_STATUS_PARTIAL_DONE	BIT(4)
 193#define SSS_HASH_STATUS_BUFFER_READY	BIT(0)
 194
 195#define SSS_REG_HASH_MSG_SIZE_LOW	0x20
 196#define SSS_REG_HASH_MSG_SIZE_HIGH	0x24
 197
 198#define SSS_REG_HASH_PRE_MSG_SIZE_LOW	0x28
 199#define SSS_REG_HASH_PRE_MSG_SIZE_HIGH	0x2C
 200
 201#define SSS_REG_HASH_IV(s)		(0xB0 + ((s) << 2))
 202#define SSS_REG_HASH_OUT(s)		(0x100 + ((s) << 2))
 203
 204#define HASH_BLOCK_SIZE			64
 205#define HASH_REG_SIZEOF			4
 206#define HASH_MD5_MAX_REG		(MD5_DIGEST_SIZE / HASH_REG_SIZEOF)
 207#define HASH_SHA1_MAX_REG		(SHA1_DIGEST_SIZE / HASH_REG_SIZEOF)
 208#define HASH_SHA256_MAX_REG		(SHA256_DIGEST_SIZE / HASH_REG_SIZEOF)
 209
 210/*
 211 * HASH bit numbers, used by device, setting in dev->hash_flags with
 212 * functions set_bit(), clear_bit() or tested with test_bit() or BIT(),
 213 * to keep HASH state BUSY or FREE, or to signal state from irq_handler
 214 * to hash_tasklet. SGS keep track of allocated memory for scatterlist
 215 */
 216#define HASH_FLAGS_BUSY		0
 217#define HASH_FLAGS_FINAL	1
 218#define HASH_FLAGS_DMA_ACTIVE	2
 219#define HASH_FLAGS_OUTPUT_READY	3
 220#define HASH_FLAGS_DMA_READY	4
 221#define HASH_FLAGS_SGS_COPIED	5
 222#define HASH_FLAGS_SGS_ALLOCED	6
 223
 224/* HASH HW constants */
 225#define BUFLEN			HASH_BLOCK_SIZE
 226
 
 
 
 227#define SSS_HASH_QUEUE_LENGTH	10
 228
 229/**
 230 * struct samsung_aes_variant - platform specific SSS driver data
 231 * @aes_offset: AES register offset from SSS module's base.
 232 * @hash_offset: HASH register offset from SSS module's base.
 233 * @clk_names: names of clocks needed to run SSS IP
 234 *
 235 * Specifies platform specific configuration of SSS module.
 236 * Note: A structure for driver specific platform data is used for future
 237 * expansion of its usage.
 238 */
 239struct samsung_aes_variant {
 240	unsigned int			aes_offset;
 241	unsigned int			hash_offset;
 242	const char			*clk_names[2];
 243};
 244
 245struct s5p_aes_reqctx {
 246	unsigned long			mode;
 247};
 248
 249struct s5p_aes_ctx {
 250	struct s5p_aes_dev		*dev;
 251
 252	u8				aes_key[AES_MAX_KEY_SIZE];
 253	u8				nonce[CTR_RFC3686_NONCE_SIZE];
 254	int				keylen;
 255};
 256
 257/**
 258 * struct s5p_aes_dev - Crypto device state container
 259 * @dev:	Associated device
 260 * @clk:	Clock for accessing hardware
 261 * @pclk:	APB bus clock necessary to access the hardware
 262 * @ioaddr:	Mapped IO memory region
 263 * @aes_ioaddr:	Per-varian offset for AES block IO memory
 264 * @irq_fc:	Feed control interrupt line
 265 * @req:	Crypto request currently handled by the device
 266 * @ctx:	Configuration for currently handled crypto request
 267 * @sg_src:	Scatter list with source data for currently handled block
 268 *		in device.  This is DMA-mapped into device.
 269 * @sg_dst:	Scatter list with destination data for currently handled block
 270 *		in device. This is DMA-mapped into device.
 271 * @sg_src_cpy:	In case of unaligned access, copied scatter list
 272 *		with source data.
 273 * @sg_dst_cpy:	In case of unaligned access, copied scatter list
 274 *		with destination data.
 275 * @tasklet:	New request scheduling jib
 276 * @queue:	Crypto queue
 277 * @busy:	Indicates whether the device is currently handling some request
 278 *		thus it uses some of the fields from this state, like:
 279 *		req, ctx, sg_src/dst (and copies).  This essentially
 280 *		protects against concurrent access to these fields.
 281 * @lock:	Lock for protecting both access to device hardware registers
 282 *		and fields related to current request (including the busy field).
 283 * @res:	Resources for hash.
 284 * @io_hash_base: Per-variant offset for HASH block IO memory.
 285 * @hash_lock:	Lock for protecting hash_req, hash_queue and hash_flags
 286 *		variable.
 287 * @hash_flags:	Flags for current HASH op.
 288 * @hash_queue:	Async hash queue.
 289 * @hash_tasklet: New HASH request scheduling job.
 290 * @xmit_buf:	Buffer for current HASH request transfer into SSS block.
 291 * @hash_req:	Current request sending to SSS HASH block.
 292 * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block.
 293 * @hash_sg_cnt: Counter for hash_sg_iter.
 294 *
 295 * @use_hash:	true if HASH algs enabled
 296 */
 297struct s5p_aes_dev {
 298	struct device			*dev;
 299	struct clk			*clk;
 300	struct clk			*pclk;
 301	void __iomem			*ioaddr;
 302	void __iomem			*aes_ioaddr;
 303	int				irq_fc;
 304
 305	struct skcipher_request		*req;
 306	struct s5p_aes_ctx		*ctx;
 307	struct scatterlist		*sg_src;
 308	struct scatterlist		*sg_dst;
 309
 310	struct scatterlist		*sg_src_cpy;
 311	struct scatterlist		*sg_dst_cpy;
 312
 313	struct tasklet_struct		tasklet;
 314	struct crypto_queue		queue;
 315	bool				busy;
 316	spinlock_t			lock;
 317
 318	struct resource			*res;
 319	void __iomem			*io_hash_base;
 320
 321	spinlock_t			hash_lock; /* protect hash_ vars */
 322	unsigned long			hash_flags;
 323	struct crypto_queue		hash_queue;
 324	struct tasklet_struct		hash_tasklet;
 325
 326	u8				xmit_buf[BUFLEN];
 327	struct ahash_request		*hash_req;
 328	struct scatterlist		*hash_sg_iter;
 329	unsigned int			hash_sg_cnt;
 330
 331	bool				use_hash;
 332};
 333
 334/**
 335 * struct s5p_hash_reqctx - HASH request context
 336 * @dd:		Associated device
 337 * @op_update:	Current request operation (OP_UPDATE or OP_FINAL)
 338 * @digcnt:	Number of bytes processed by HW (without buffer[] ones)
 339 * @digest:	Digest message or IV for partial result
 340 * @nregs:	Number of HW registers for digest or IV read/write
 341 * @engine:	Bits for selecting type of HASH in SSS block
 342 * @sg:		sg for DMA transfer
 343 * @sg_len:	Length of sg for DMA transfer
 344 * @sgl:	sg for joining buffer and req->src scatterlist
 345 * @skip:	Skip offset in req->src for current op
 346 * @total:	Total number of bytes for current request
 347 * @finup:	Keep state for finup or final.
 348 * @error:	Keep track of error.
 349 * @bufcnt:	Number of bytes holded in buffer[]
 350 * @buffer:	For byte(s) from end of req->src in UPDATE op
 351 */
 352struct s5p_hash_reqctx {
 353	struct s5p_aes_dev	*dd;
 354	bool			op_update;
 355
 356	u64			digcnt;
 357	u8			digest[SHA256_DIGEST_SIZE];
 358
 359	unsigned int		nregs; /* digest_size / sizeof(reg) */
 360	u32			engine;
 361
 362	struct scatterlist	*sg;
 363	unsigned int		sg_len;
 364	struct scatterlist	sgl[2];
 365	unsigned int		skip;
 366	unsigned int		total;
 367	bool			finup;
 368	bool			error;
 369
 370	u32			bufcnt;
 371	u8			buffer[];
 372};
 373
 374/**
 375 * struct s5p_hash_ctx - HASH transformation context
 376 * @dd:		Associated device
 377 * @flags:	Bits for algorithm HASH.
 378 * @fallback:	Software transformation for zero message or size < BUFLEN.
 379 */
 380struct s5p_hash_ctx {
 381	struct s5p_aes_dev	*dd;
 382	unsigned long		flags;
 383	struct crypto_shash	*fallback;
 384};
 385
 386static const struct samsung_aes_variant s5p_aes_data = {
 387	.aes_offset	= 0x4000,
 388	.hash_offset	= 0x6000,
 389	.clk_names	= { "secss", },
 390};
 391
 392static const struct samsung_aes_variant exynos_aes_data = {
 393	.aes_offset	= 0x200,
 394	.hash_offset	= 0x400,
 395	.clk_names	= { "secss", },
 396};
 397
 398static const struct samsung_aes_variant exynos5433_slim_aes_data = {
 399	.aes_offset	= 0x400,
 400	.hash_offset	= 0x800,
 401	.clk_names	= { "aclk", "pclk", },
 402};
 403
 404static const struct of_device_id s5p_sss_dt_match[] = {
 405	{
 406		.compatible = "samsung,s5pv210-secss",
 407		.data = &s5p_aes_data,
 408	},
 409	{
 410		.compatible = "samsung,exynos4210-secss",
 411		.data = &exynos_aes_data,
 412	},
 413	{
 414		.compatible = "samsung,exynos5433-slim-sss",
 415		.data = &exynos5433_slim_aes_data,
 416	},
 417	{ },
 418};
 419MODULE_DEVICE_TABLE(of, s5p_sss_dt_match);
 420
 421static inline const struct samsung_aes_variant *find_s5p_sss_version
 422				   (const struct platform_device *pdev)
 423{
 424	if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node))
 425		return of_device_get_match_data(&pdev->dev);
 426
 427	return (const struct samsung_aes_variant *)
 428			platform_get_device_id(pdev)->driver_data;
 429}
 430
 431static struct s5p_aes_dev *s5p_dev;
 432
 433static void s5p_set_dma_indata(struct s5p_aes_dev *dev,
 434			       const struct scatterlist *sg)
 435{
 436	SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));
 437	SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));
 438}
 439
 440static void s5p_set_dma_outdata(struct s5p_aes_dev *dev,
 441				const struct scatterlist *sg)
 442{
 443	SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));
 444	SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));
 445}
 446
 447static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg)
 448{
 449	int len;
 450
 451	if (!*sg)
 452		return;
 453
 454	len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE);
 455	free_pages((unsigned long)sg_virt(*sg), get_order(len));
 456
 457	kfree(*sg);
 458	*sg = NULL;
 459}
 460
 461static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg,
 462			    unsigned int nbytes, int out)
 463{
 464	struct scatter_walk walk;
 465
 466	if (!nbytes)
 467		return;
 468
 469	scatterwalk_start(&walk, sg);
 470	scatterwalk_copychunks(buf, &walk, nbytes, out);
 471	scatterwalk_done(&walk, out, 0);
 472}
 473
 474static void s5p_sg_done(struct s5p_aes_dev *dev)
 475{
 476	struct skcipher_request *req = dev->req;
 477	struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req);
 478
 479	if (dev->sg_dst_cpy) {
 480		dev_dbg(dev->dev,
 481			"Copying %d bytes of output data back to original place\n",
 482			dev->req->cryptlen);
 483		s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst,
 484				dev->req->cryptlen, 1);
 485	}
 486	s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
 487	s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
 488	if (reqctx->mode & FLAGS_AES_CBC)
 489		memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), AES_BLOCK_SIZE);
 490
 491	else if (reqctx->mode & FLAGS_AES_CTR)
 492		memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), AES_BLOCK_SIZE);
 493}
 494
 495/* Calls the completion. Cannot be called with dev->lock hold. */
 496static void s5p_aes_complete(struct skcipher_request *req, int err)
 497{
 498	skcipher_request_complete(req, err);
 499}
 500
 501static void s5p_unset_outdata(struct s5p_aes_dev *dev)
 502{
 503	dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);
 504}
 505
 506static void s5p_unset_indata(struct s5p_aes_dev *dev)
 507{
 508	dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);
 509}
 510
 511static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
 512			   struct scatterlist **dst)
 513{
 514	void *pages;
 515	int len;
 516
 517	*dst = kmalloc(sizeof(**dst), GFP_ATOMIC);
 518	if (!*dst)
 519		return -ENOMEM;
 520
 521	len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE);
 522	pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len));
 523	if (!pages) {
 524		kfree(*dst);
 525		*dst = NULL;
 526		return -ENOMEM;
 527	}
 528
 529	s5p_sg_copy_buf(pages, src, dev->req->cryptlen, 0);
 530
 531	sg_init_table(*dst, 1);
 532	sg_set_buf(*dst, pages, len);
 533
 534	return 0;
 535}
 536
 537static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
 538{
 539	if (!sg->length)
 540		return -EINVAL;
 541
 542	if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE))
 543		return -ENOMEM;
 544
 545	dev->sg_dst = sg;
 546
 547	return 0;
 548}
 549
 550static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
 551{
 552	if (!sg->length)
 553		return -EINVAL;
 554
 555	if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE))
 556		return -ENOMEM;
 557
 558	dev->sg_src = sg;
 559
 560	return 0;
 561}
 562
 563/*
 564 * Returns -ERRNO on error (mapping of new data failed).
 565 * On success returns:
 566 *  - 0 if there is no more data,
 567 *  - 1 if new transmitting (output) data is ready and its address+length
 568 *     have to be written to device (by calling s5p_set_dma_outdata()).
 569 */
 570static int s5p_aes_tx(struct s5p_aes_dev *dev)
 571{
 572	int ret = 0;
 573
 574	s5p_unset_outdata(dev);
 575
 576	if (!sg_is_last(dev->sg_dst)) {
 577		ret = s5p_set_outdata(dev, sg_next(dev->sg_dst));
 578		if (!ret)
 579			ret = 1;
 580	}
 581
 582	return ret;
 583}
 584
 585/*
 586 * Returns -ERRNO on error (mapping of new data failed).
 587 * On success returns:
 588 *  - 0 if there is no more data,
 589 *  - 1 if new receiving (input) data is ready and its address+length
 590 *     have to be written to device (by calling s5p_set_dma_indata()).
 591 */
 592static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/)
 593{
 594	int ret = 0;
 595
 596	s5p_unset_indata(dev);
 597
 598	if (!sg_is_last(dev->sg_src)) {
 599		ret = s5p_set_indata(dev, sg_next(dev->sg_src));
 600		if (!ret)
 601			ret = 1;
 602	}
 603
 604	return ret;
 605}
 606
 607static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset)
 608{
 609	return __raw_readl(dd->io_hash_base + offset);
 610}
 611
 612static inline void s5p_hash_write(struct s5p_aes_dev *dd,
 613				  u32 offset, u32 value)
 614{
 615	__raw_writel(value, dd->io_hash_base + offset);
 616}
 617
 618/**
 619 * s5p_set_dma_hashdata() - start DMA with sg
 620 * @dev:	device
 621 * @sg:		scatterlist ready to DMA transmit
 622 */
 623static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev,
 624				 const struct scatterlist *sg)
 625{
 626	dev->hash_sg_cnt--;
 627	SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg));
 628	SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */
 629}
 630
 631/**
 632 * s5p_hash_rx() - get next hash_sg_iter
 633 * @dev:	device
 634 *
 635 * Return:
 636 * 2	if there is no more data and it is UPDATE op
 637 * 1	if new receiving (input) data is ready and can be written to device
 638 * 0	if there is no more data and it is FINAL op
 639 */
 640static int s5p_hash_rx(struct s5p_aes_dev *dev)
 641{
 642	if (dev->hash_sg_cnt > 0) {
 643		dev->hash_sg_iter = sg_next(dev->hash_sg_iter);
 644		return 1;
 645	}
 646
 647	set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags);
 648	if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags))
 649		return 0;
 650
 651	return 2;
 652}
 653
 654static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
 655{
 656	struct platform_device *pdev = dev_id;
 657	struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
 658	struct skcipher_request *req;
 659	int err_dma_tx = 0;
 660	int err_dma_rx = 0;
 661	int err_dma_hx = 0;
 662	bool tx_end = false;
 663	bool hx_end = false;
 664	unsigned long flags;
 665	u32 status, st_bits;
 666	int err;
 667
 668	spin_lock_irqsave(&dev->lock, flags);
 669
 670	/*
 671	 * Handle rx or tx interrupt. If there is still data (scatterlist did not
 672	 * reach end), then map next scatterlist entry.
 673	 * In case of such mapping error, s5p_aes_complete() should be called.
 674	 *
 675	 * If there is no more data in tx scatter list, call s5p_aes_complete()
 676	 * and schedule new tasklet.
 677	 *
 678	 * Handle hx interrupt. If there is still data map next entry.
 679	 */
 680	status = SSS_READ(dev, FCINTSTAT);
 681	if (status & SSS_FCINTSTAT_BRDMAINT)
 682		err_dma_rx = s5p_aes_rx(dev);
 683
 684	if (status & SSS_FCINTSTAT_BTDMAINT) {
 685		if (sg_is_last(dev->sg_dst))
 686			tx_end = true;
 687		err_dma_tx = s5p_aes_tx(dev);
 688	}
 689
 690	if (status & SSS_FCINTSTAT_HRDMAINT)
 691		err_dma_hx = s5p_hash_rx(dev);
 692
 693	st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT |
 694				SSS_FCINTSTAT_HRDMAINT);
 695	/* clear DMA bits */
 696	SSS_WRITE(dev, FCINTPEND, st_bits);
 697
 698	/* clear HASH irq bits */
 699	if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) {
 700		/* cannot have both HPART and HDONE */
 701		if (status & SSS_FCINTSTAT_HPARTINT)
 702			st_bits = SSS_HASH_STATUS_PARTIAL_DONE;
 703
 704		if (status & SSS_FCINTSTAT_HDONEINT)
 705			st_bits = SSS_HASH_STATUS_MSG_DONE;
 706
 707		set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags);
 708		s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits);
 709		hx_end = true;
 710		/* when DONE or PART, do not handle HASH DMA */
 711		err_dma_hx = 0;
 712	}
 713
 714	if (err_dma_rx < 0) {
 715		err = err_dma_rx;
 716		goto error;
 717	}
 718	if (err_dma_tx < 0) {
 719		err = err_dma_tx;
 720		goto error;
 721	}
 722
 723	if (tx_end) {
 724		s5p_sg_done(dev);
 725		if (err_dma_hx == 1)
 726			s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
 727
 728		spin_unlock_irqrestore(&dev->lock, flags);
 729
 730		s5p_aes_complete(dev->req, 0);
 731		/* Device is still busy */
 732		tasklet_schedule(&dev->tasklet);
 733	} else {
 734		/*
 735		 * Writing length of DMA block (either receiving or
 736		 * transmitting) will start the operation immediately, so this
 737		 * should be done at the end (even after clearing pending
 738		 * interrupts to not miss the interrupt).
 739		 */
 740		if (err_dma_tx == 1)
 741			s5p_set_dma_outdata(dev, dev->sg_dst);
 742		if (err_dma_rx == 1)
 743			s5p_set_dma_indata(dev, dev->sg_src);
 744		if (err_dma_hx == 1)
 745			s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
 746
 747		spin_unlock_irqrestore(&dev->lock, flags);
 748	}
 749
 750	goto hash_irq_end;
 751
 752error:
 753	s5p_sg_done(dev);
 754	dev->busy = false;
 755	req = dev->req;
 756	if (err_dma_hx == 1)
 757		s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
 758
 759	spin_unlock_irqrestore(&dev->lock, flags);
 760	s5p_aes_complete(req, err);
 761
 762hash_irq_end:
 763	/*
 764	 * Note about else if:
 765	 *   when hash_sg_iter reaches end and its UPDATE op,
 766	 *   issue SSS_HASH_PAUSE and wait for HPART irq
 767	 */
 768	if (hx_end)
 769		tasklet_schedule(&dev->hash_tasklet);
 770	else if (err_dma_hx == 2)
 771		s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE,
 772			       SSS_HASH_PAUSE);
 773
 774	return IRQ_HANDLED;
 775}
 776
 777/**
 778 * s5p_hash_read_msg() - read message or IV from HW
 779 * @req:	AHASH request
 780 */
 781static void s5p_hash_read_msg(struct ahash_request *req)
 782{
 783	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
 784	struct s5p_aes_dev *dd = ctx->dd;
 785	u32 *hash = (u32 *)ctx->digest;
 786	unsigned int i;
 787
 788	for (i = 0; i < ctx->nregs; i++)
 789		hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i));
 790}
 791
 792/**
 793 * s5p_hash_write_ctx_iv() - write IV for next partial/finup op.
 794 * @dd:		device
 795 * @ctx:	request context
 796 */
 797static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd,
 798				  const struct s5p_hash_reqctx *ctx)
 799{
 800	const u32 *hash = (const u32 *)ctx->digest;
 801	unsigned int i;
 802
 803	for (i = 0; i < ctx->nregs; i++)
 804		s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]);
 805}
 806
 807/**
 808 * s5p_hash_write_iv() - write IV for next partial/finup op.
 809 * @req:	AHASH request
 810 */
 811static void s5p_hash_write_iv(struct ahash_request *req)
 812{
 813	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
 814
 815	s5p_hash_write_ctx_iv(ctx->dd, ctx);
 816}
 817
 818/**
 819 * s5p_hash_copy_result() - copy digest into req->result
 820 * @req:	AHASH request
 821 */
 822static void s5p_hash_copy_result(struct ahash_request *req)
 823{
 824	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
 825
 826	if (!req->result)
 827		return;
 828
 829	memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF);
 830}
 831
 832/**
 833 * s5p_hash_dma_flush() - flush HASH DMA
 834 * @dev:	secss device
 835 */
 836static void s5p_hash_dma_flush(struct s5p_aes_dev *dev)
 837{
 838	SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH);
 839}
 840
 841/**
 842 * s5p_hash_dma_enable() - enable DMA mode for HASH
 843 * @dev:	secss device
 844 *
 845 * enable DMA mode for HASH
 846 */
 847static void s5p_hash_dma_enable(struct s5p_aes_dev *dev)
 848{
 849	s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA);
 850}
 851
 852/**
 853 * s5p_hash_irq_disable() - disable irq HASH signals
 854 * @dev:	secss device
 855 * @flags:	bitfield with irq's to be disabled
 856 */
 857static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags)
 858{
 859	SSS_WRITE(dev, FCINTENCLR, flags);
 860}
 861
 862/**
 863 * s5p_hash_irq_enable() - enable irq signals
 864 * @dev:	secss device
 865 * @flags:	bitfield with irq's to be enabled
 866 */
 867static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags)
 868{
 869	SSS_WRITE(dev, FCINTENSET, flags);
 870}
 871
 872/**
 873 * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH
 874 * @dev:	secss device
 875 * @hashflow:	HASH stream flow with/without crypto AES/DES
 876 */
 877static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow)
 878{
 879	unsigned long flags;
 880	u32 flow;
 881
 882	spin_lock_irqsave(&dev->lock, flags);
 883
 884	flow = SSS_READ(dev, FCFIFOCTRL);
 885	flow &= ~SSS_HASHIN_MASK;
 886	flow |= hashflow;
 887	SSS_WRITE(dev, FCFIFOCTRL, flow);
 888
 889	spin_unlock_irqrestore(&dev->lock, flags);
 890}
 891
 892/**
 893 * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS
 894 * @dev:	secss device
 895 * @hashflow:	HASH stream flow with/without AES/DES
 896 *
 897 * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW,
 898 * enable HASH irq's HRDMA, HDONE, HPART
 899 */
 900static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow)
 901{
 902	s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR |
 903			     SSS_FCINTENCLR_HDONEINTENCLR |
 904			     SSS_FCINTENCLR_HPARTINTENCLR);
 905	s5p_hash_dma_flush(dev);
 906
 907	s5p_hash_dma_enable(dev);
 908	s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK);
 909	s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET |
 910			    SSS_FCINTENSET_HDONEINTENSET |
 911			    SSS_FCINTENSET_HPARTINTENSET);
 912}
 913
 914/**
 915 * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing
 916 * @dd:		secss device
 917 * @length:	length for request
 918 * @final:	true if final op
 919 *
 920 * Prepare SSS HASH block for processing bytes in DMA mode. If it is called
 921 * after previous updates, fill up IV words. For final, calculate and set
 922 * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH
 923 * length as 2^63 so it will be never reached and set to zero prelow and
 924 * prehigh.
 925 *
 926 * This function does not start DMA transfer.
 927 */
 928static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length,
 929				bool final)
 930{
 931	struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
 932	u32 prelow, prehigh, low, high;
 933	u32 configflags, swapflags;
 934	u64 tmplen;
 935
 936	configflags = ctx->engine | SSS_HASH_INIT_BIT;
 937
 938	if (likely(ctx->digcnt)) {
 939		s5p_hash_write_ctx_iv(dd, ctx);
 940		configflags |= SSS_HASH_USER_IV_EN;
 941	}
 942
 943	if (final) {
 944		/* number of bytes for last part */
 945		low = length;
 946		high = 0;
 947		/* total number of bits prev hashed */
 948		tmplen = ctx->digcnt * 8;
 949		prelow = (u32)tmplen;
 950		prehigh = (u32)(tmplen >> 32);
 951	} else {
 952		prelow = 0;
 953		prehigh = 0;
 954		low = 0;
 955		high = BIT(31);
 956	}
 957
 958	swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO |
 959		    SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY;
 960
 961	s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low);
 962	s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high);
 963	s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow);
 964	s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh);
 965
 966	s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags);
 967	s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags);
 968}
 969
 970/**
 971 * s5p_hash_xmit_dma() - start DMA hash processing
 972 * @dd:		secss device
 973 * @length:	length for request
 974 * @final:	true if final op
 975 *
 976 * Update digcnt here, as it is needed for finup/final op.
 977 */
 978static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length,
 979			     bool final)
 980{
 981	struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
 982	unsigned int cnt;
 983
 984	cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
 985	if (!cnt) {
 986		dev_err(dd->dev, "dma_map_sg error\n");
 987		ctx->error = true;
 988		return -EINVAL;
 989	}
 990
 991	set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
 992	dd->hash_sg_iter = ctx->sg;
 993	dd->hash_sg_cnt = cnt;
 994	s5p_hash_write_ctrl(dd, length, final);
 995	ctx->digcnt += length;
 996	ctx->total -= length;
 997
 998	/* catch last interrupt */
 999	if (final)
1000		set_bit(HASH_FLAGS_FINAL, &dd->hash_flags);
1001
1002	s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */
1003
1004	return -EINPROGRESS;
1005}
1006
1007/**
1008 * s5p_hash_copy_sgs() - copy request's bytes into new buffer
1009 * @ctx:	request context
1010 * @sg:		source scatterlist request
1011 * @new_len:	number of bytes to process from sg
1012 *
1013 * Allocate new buffer, copy data for HASH into it. If there was xmit_buf
1014 * filled, copy it first, then copy data from sg into it. Prepare one sgl[0]
1015 * with allocated buffer.
1016 *
1017 * Set bit in dd->hash_flag so we can free it after irq ends processing.
1018 */
1019static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx,
1020			     struct scatterlist *sg, unsigned int new_len)
1021{
1022	unsigned int pages, len;
1023	void *buf;
1024
1025	len = new_len + ctx->bufcnt;
1026	pages = get_order(len);
1027
1028	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
1029	if (!buf) {
1030		dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n");
1031		ctx->error = true;
1032		return -ENOMEM;
1033	}
1034
1035	if (ctx->bufcnt)
1036		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
1037
1038	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip,
1039				 new_len, 0);
1040	sg_init_table(ctx->sgl, 1);
1041	sg_set_buf(ctx->sgl, buf, len);
1042	ctx->sg = ctx->sgl;
1043	ctx->sg_len = 1;
1044	ctx->bufcnt = 0;
1045	ctx->skip = 0;
1046	set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags);
1047
1048	return 0;
1049}
1050
1051/**
1052 * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy
1053 * @ctx:	request context
1054 * @sg:		source scatterlist request
1055 * @new_len:	number of bytes to process from sg
1056 *
1057 * Allocate new scatterlist table, copy data for HASH into it. If there was
1058 * xmit_buf filled, prepare it first, then copy page, length and offset from
1059 * source sg into it, adjusting begin and/or end for skip offset and
1060 * hash_later value.
1061 *
1062 * Resulting sg table will be assigned to ctx->sg. Set flag so we can free
1063 * it after irq ends processing.
1064 */
1065static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx,
1066				  struct scatterlist *sg, unsigned int new_len)
1067{
1068	unsigned int skip = ctx->skip, n = sg_nents(sg);
1069	struct scatterlist *tmp;
1070	unsigned int len;
1071
1072	if (ctx->bufcnt)
1073		n++;
1074
1075	ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
1076	if (!ctx->sg) {
1077		ctx->error = true;
1078		return -ENOMEM;
1079	}
1080
1081	sg_init_table(ctx->sg, n);
1082
1083	tmp = ctx->sg;
1084
1085	ctx->sg_len = 0;
1086
1087	if (ctx->bufcnt) {
1088		sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
1089		tmp = sg_next(tmp);
1090		ctx->sg_len++;
1091	}
1092
1093	while (sg && skip >= sg->length) {
1094		skip -= sg->length;
1095		sg = sg_next(sg);
1096	}
1097
1098	while (sg && new_len) {
1099		len = sg->length - skip;
1100		if (new_len < len)
1101			len = new_len;
1102
1103		new_len -= len;
1104		sg_set_page(tmp, sg_page(sg), len, sg->offset + skip);
1105		skip = 0;
1106		if (new_len <= 0)
1107			sg_mark_end(tmp);
1108
1109		tmp = sg_next(tmp);
1110		ctx->sg_len++;
1111		sg = sg_next(sg);
1112	}
1113
1114	set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags);
1115
1116	return 0;
1117}
1118
1119/**
1120 * s5p_hash_prepare_sgs() - prepare sg for processing
1121 * @ctx:	request context
1122 * @sg:		source scatterlist request
1123 * @new_len:	number of bytes to process from sg
1124 * @final:	final flag
1125 *
1126 * Check two conditions: (1) if buffers in sg have len aligned data, and (2)
1127 * sg table have good aligned elements (list_ok). If one of this checks fails,
1128 * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy
1129 * data into this buffer and prepare request in sgl, or (2) allocates new sg
1130 * table and prepare sg elements.
1131 *
1132 * For digest or finup all conditions can be good, and we may not need any
1133 * fixes.
1134 */
1135static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx,
1136				struct scatterlist *sg,
1137				unsigned int new_len, bool final)
1138{
1139	unsigned int skip = ctx->skip, nbytes = new_len, n = 0;
1140	bool aligned = true, list_ok = true;
1141	struct scatterlist *sg_tmp = sg;
1142
1143	if (!sg || !sg->length || !new_len)
1144		return 0;
1145
1146	if (skip || !final)
1147		list_ok = false;
1148
1149	while (nbytes > 0 && sg_tmp) {
1150		n++;
1151		if (skip >= sg_tmp->length) {
1152			skip -= sg_tmp->length;
1153			if (!sg_tmp->length) {
1154				aligned = false;
1155				break;
1156			}
1157		} else {
1158			if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) {
1159				aligned = false;
1160				break;
1161			}
1162
1163			if (nbytes < sg_tmp->length - skip) {
1164				list_ok = false;
1165				break;
1166			}
1167
1168			nbytes -= sg_tmp->length - skip;
1169			skip = 0;
1170		}
1171
1172		sg_tmp = sg_next(sg_tmp);
1173	}
1174
1175	if (!aligned)
1176		return s5p_hash_copy_sgs(ctx, sg, new_len);
1177	else if (!list_ok)
1178		return s5p_hash_copy_sg_lists(ctx, sg, new_len);
1179
1180	/*
1181	 * Have aligned data from previous operation and/or current
1182	 * Note: will enter here only if (digest or finup) and aligned
1183	 */
1184	if (ctx->bufcnt) {
1185		ctx->sg_len = n;
1186		sg_init_table(ctx->sgl, 2);
1187		sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt);
1188		sg_chain(ctx->sgl, 2, sg);
1189		ctx->sg = ctx->sgl;
1190		ctx->sg_len++;
1191	} else {
1192		ctx->sg = sg;
1193		ctx->sg_len = n;
1194	}
1195
1196	return 0;
1197}
1198
1199/**
1200 * s5p_hash_prepare_request() - prepare request for processing
1201 * @req:	AHASH request
1202 * @update:	true if UPDATE op
1203 *
1204 * Note 1: we can have update flag _and_ final flag at the same time.
1205 * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or
1206 *	   either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or
1207 *	   we have final op
1208 */
1209static int s5p_hash_prepare_request(struct ahash_request *req, bool update)
1210{
1211	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1212	bool final = ctx->finup;
1213	int xmit_len, hash_later, nbytes;
1214	int ret;
1215
1216	if (update)
1217		nbytes = req->nbytes;
1218	else
1219		nbytes = 0;
1220
1221	ctx->total = nbytes + ctx->bufcnt;
1222	if (!ctx->total)
1223		return 0;
1224
1225	if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) {
1226		/* bytes left from previous request, so fill up to BUFLEN */
1227		int len = BUFLEN - ctx->bufcnt % BUFLEN;
1228
1229		if (len > nbytes)
1230			len = nbytes;
1231
1232		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1233					 0, len, 0);
1234		ctx->bufcnt += len;
1235		nbytes -= len;
1236		ctx->skip = len;
1237	} else {
1238		ctx->skip = 0;
1239	}
1240
1241	if (ctx->bufcnt)
1242		memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt);
1243
1244	xmit_len = ctx->total;
1245	if (final) {
1246		hash_later = 0;
1247	} else {
1248		if (IS_ALIGNED(xmit_len, BUFLEN))
1249			xmit_len -= BUFLEN;
1250		else
1251			xmit_len -= xmit_len & (BUFLEN - 1);
1252
1253		hash_later = ctx->total - xmit_len;
1254		/* copy hash_later bytes from end of req->src */
1255		/* previous bytes are in xmit_buf, so no overwrite */
1256		scatterwalk_map_and_copy(ctx->buffer, req->src,
1257					 req->nbytes - hash_later,
1258					 hash_later, 0);
1259	}
1260
1261	if (xmit_len > BUFLEN) {
1262		ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later,
1263					   final);
1264		if (ret)
1265			return ret;
1266	} else {
1267		/* have buffered data only */
1268		if (unlikely(!ctx->bufcnt)) {
1269			/* first update didn't fill up buffer */
1270			scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src,
1271						 0, xmit_len, 0);
1272		}
1273
1274		sg_init_table(ctx->sgl, 1);
1275		sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len);
1276
1277		ctx->sg = ctx->sgl;
1278		ctx->sg_len = 1;
1279	}
1280
1281	ctx->bufcnt = hash_later;
1282	if (!final)
1283		ctx->total = xmit_len;
1284
1285	return 0;
1286}
1287
1288/**
1289 * s5p_hash_update_dma_stop() - unmap DMA
1290 * @dd:		secss device
1291 *
1292 * Unmap scatterlist ctx->sg.
1293 */
1294static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd)
1295{
1296	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
1297
1298	dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
1299	clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
1300}
1301
1302/**
1303 * s5p_hash_finish() - copy calculated digest to crypto layer
1304 * @req:	AHASH request
1305 */
1306static void s5p_hash_finish(struct ahash_request *req)
1307{
1308	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1309	struct s5p_aes_dev *dd = ctx->dd;
1310
1311	if (ctx->digcnt)
1312		s5p_hash_copy_result(req);
1313
1314	dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt);
1315}
1316
1317/**
1318 * s5p_hash_finish_req() - finish request
1319 * @req:	AHASH request
1320 * @err:	error
1321 */
1322static void s5p_hash_finish_req(struct ahash_request *req, int err)
1323{
1324	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1325	struct s5p_aes_dev *dd = ctx->dd;
1326	unsigned long flags;
1327
1328	if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags))
1329		free_pages((unsigned long)sg_virt(ctx->sg),
1330			   get_order(ctx->sg->length));
1331
1332	if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags))
1333		kfree(ctx->sg);
1334
1335	ctx->sg = NULL;
1336	dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) |
1337			    BIT(HASH_FLAGS_SGS_COPIED));
1338
1339	if (!err && !ctx->error) {
1340		s5p_hash_read_msg(req);
1341		if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags))
1342			s5p_hash_finish(req);
1343	} else {
1344		ctx->error = true;
1345	}
1346
1347	spin_lock_irqsave(&dd->hash_lock, flags);
1348	dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) |
1349			    BIT(HASH_FLAGS_DMA_READY) |
1350			    BIT(HASH_FLAGS_OUTPUT_READY));
1351	spin_unlock_irqrestore(&dd->hash_lock, flags);
1352
1353	if (req->base.complete)
1354		ahash_request_complete(req, err);
1355}
1356
1357/**
1358 * s5p_hash_handle_queue() - handle hash queue
1359 * @dd:		device s5p_aes_dev
1360 * @req:	AHASH request
1361 *
1362 * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the
1363 * device then processes the first request from the dd->queue
1364 *
1365 * Returns: see s5p_hash_final below.
1366 */
1367static int s5p_hash_handle_queue(struct s5p_aes_dev *dd,
1368				 struct ahash_request *req)
1369{
1370	struct crypto_async_request *async_req, *backlog;
1371	struct s5p_hash_reqctx *ctx;
1372	unsigned long flags;
1373	int err = 0, ret = 0;
1374
1375retry:
1376	spin_lock_irqsave(&dd->hash_lock, flags);
1377	if (req)
1378		ret = ahash_enqueue_request(&dd->hash_queue, req);
1379
1380	if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1381		spin_unlock_irqrestore(&dd->hash_lock, flags);
1382		return ret;
1383	}
1384
1385	backlog = crypto_get_backlog(&dd->hash_queue);
1386	async_req = crypto_dequeue_request(&dd->hash_queue);
1387	if (async_req)
1388		set_bit(HASH_FLAGS_BUSY, &dd->hash_flags);
1389
1390	spin_unlock_irqrestore(&dd->hash_lock, flags);
1391
1392	if (!async_req)
1393		return ret;
1394
1395	if (backlog)
1396		crypto_request_complete(backlog, -EINPROGRESS);
1397
1398	req = ahash_request_cast(async_req);
1399	dd->hash_req = req;
1400	ctx = ahash_request_ctx(req);
1401
1402	err = s5p_hash_prepare_request(req, ctx->op_update);
1403	if (err || !ctx->total)
1404		goto out;
1405
1406	dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n",
1407		ctx->op_update, req->nbytes);
1408
1409	s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT);
1410	if (ctx->digcnt)
1411		s5p_hash_write_iv(req); /* restore hash IV */
1412
1413	if (ctx->op_update) { /* HASH_OP_UPDATE */
1414		err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup);
1415		if (err != -EINPROGRESS && ctx->finup && !ctx->error)
1416			/* no final() after finup() */
1417			err = s5p_hash_xmit_dma(dd, ctx->total, true);
1418	} else { /* HASH_OP_FINAL */
1419		err = s5p_hash_xmit_dma(dd, ctx->total, true);
1420	}
1421out:
1422	if (err != -EINPROGRESS) {
1423		/* hash_tasklet_cb will not finish it, so do it here */
1424		s5p_hash_finish_req(req, err);
1425		req = NULL;
1426
1427		/*
1428		 * Execute next request immediately if there is anything
1429		 * in queue.
1430		 */
1431		goto retry;
1432	}
1433
1434	return ret;
1435}
1436
1437/**
1438 * s5p_hash_tasklet_cb() - hash tasklet
1439 * @data:	ptr to s5p_aes_dev
1440 */
1441static void s5p_hash_tasklet_cb(unsigned long data)
1442{
1443	struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data;
1444
1445	if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1446		s5p_hash_handle_queue(dd, NULL);
1447		return;
1448	}
1449
1450	if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) {
1451		if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE,
1452				       &dd->hash_flags)) {
1453			s5p_hash_update_dma_stop(dd);
1454		}
1455
1456		if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY,
1457				       &dd->hash_flags)) {
1458			/* hash or semi-hash ready */
1459			clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags);
1460			goto finish;
1461		}
1462	}
1463
1464	return;
1465
1466finish:
1467	/* finish curent request */
1468	s5p_hash_finish_req(dd->hash_req, 0);
1469
1470	/* If we are not busy, process next req */
1471	if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags))
1472		s5p_hash_handle_queue(dd, NULL);
1473}
1474
1475/**
1476 * s5p_hash_enqueue() - enqueue request
1477 * @req:	AHASH request
1478 * @op:		operation UPDATE (true) or FINAL (false)
1479 *
1480 * Returns: see s5p_hash_final below.
1481 */
1482static int s5p_hash_enqueue(struct ahash_request *req, bool op)
1483{
1484	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1485	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1486
1487	ctx->op_update = op;
1488
1489	return s5p_hash_handle_queue(tctx->dd, req);
1490}
1491
1492/**
1493 * s5p_hash_update() - process the hash input data
1494 * @req:	AHASH request
1495 *
1496 * If request will fit in buffer, copy it and return immediately
1497 * else enqueue it with OP_UPDATE.
1498 *
1499 * Returns: see s5p_hash_final below.
1500 */
1501static int s5p_hash_update(struct ahash_request *req)
1502{
1503	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1504
1505	if (!req->nbytes)
1506		return 0;
1507
1508	if (ctx->bufcnt + req->nbytes <= BUFLEN) {
1509		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1510					 0, req->nbytes, 0);
1511		ctx->bufcnt += req->nbytes;
1512		return 0;
1513	}
1514
1515	return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */
1516}
1517
1518/**
1519 * s5p_hash_final() - close up hash and calculate digest
1520 * @req:	AHASH request
1521 *
1522 * Note: in final req->src do not have any data, and req->nbytes can be
1523 * non-zero.
1524 *
1525 * If there were no input data processed yet and the buffered hash data is
1526 * less than BUFLEN (64) then calculate the final hash immediately by using
1527 * SW algorithm fallback.
1528 *
1529 * Otherwise enqueues the current AHASH request with OP_FINAL operation op
1530 * and finalize hash message in HW. Note that if digcnt!=0 then there were
1531 * previous update op, so there are always some buffered bytes in ctx->buffer,
1532 * which means that ctx->bufcnt!=0
1533 *
1534 * Returns:
1535 * 0 if the request has been processed immediately,
1536 * -EINPROGRESS if the operation has been queued for later execution or is set
1537 *		to processing by HW,
1538 * -EBUSY if queue is full and request should be resubmitted later,
1539 * other negative values denotes an error.
1540 */
1541static int s5p_hash_final(struct ahash_request *req)
1542{
1543	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1544
1545	ctx->finup = true;
1546	if (ctx->error)
1547		return -EINVAL; /* uncompleted hash is not needed */
1548
1549	if (!ctx->digcnt && ctx->bufcnt < BUFLEN) {
1550		struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1551
1552		return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer,
1553					       ctx->bufcnt, req->result);
1554	}
1555
1556	return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */
1557}
1558
1559/**
1560 * s5p_hash_finup() - process last req->src and calculate digest
1561 * @req:	AHASH request containing the last update data
1562 *
1563 * Return values: see s5p_hash_final above.
1564 */
1565static int s5p_hash_finup(struct ahash_request *req)
1566{
1567	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1568	int err1, err2;
1569
1570	ctx->finup = true;
1571
1572	err1 = s5p_hash_update(req);
1573	if (err1 == -EINPROGRESS || err1 == -EBUSY)
1574		return err1;
1575
1576	/*
1577	 * final() has to be always called to cleanup resources even if
1578	 * update() failed, except EINPROGRESS or calculate digest for small
1579	 * size
1580	 */
1581	err2 = s5p_hash_final(req);
1582
1583	return err1 ?: err2;
1584}
1585
1586/**
1587 * s5p_hash_init() - initialize AHASH request contex
1588 * @req:	AHASH request
1589 *
1590 * Init async hash request context.
1591 */
1592static int s5p_hash_init(struct ahash_request *req)
1593{
1594	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1595	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1596	struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1597
1598	ctx->dd = tctx->dd;
1599	ctx->error = false;
1600	ctx->finup = false;
1601	ctx->bufcnt = 0;
1602	ctx->digcnt = 0;
1603	ctx->total = 0;
1604	ctx->skip = 0;
1605
1606	dev_dbg(tctx->dd->dev, "init: digest size: %d\n",
1607		crypto_ahash_digestsize(tfm));
1608
1609	switch (crypto_ahash_digestsize(tfm)) {
1610	case MD5_DIGEST_SIZE:
1611		ctx->engine = SSS_HASH_ENGINE_MD5;
1612		ctx->nregs = HASH_MD5_MAX_REG;
1613		break;
1614	case SHA1_DIGEST_SIZE:
1615		ctx->engine = SSS_HASH_ENGINE_SHA1;
1616		ctx->nregs = HASH_SHA1_MAX_REG;
1617		break;
1618	case SHA256_DIGEST_SIZE:
1619		ctx->engine = SSS_HASH_ENGINE_SHA256;
1620		ctx->nregs = HASH_SHA256_MAX_REG;
1621		break;
1622	default:
1623		ctx->error = true;
1624		return -EINVAL;
1625	}
1626
1627	return 0;
1628}
1629
1630/**
1631 * s5p_hash_digest - calculate digest from req->src
1632 * @req:	AHASH request
1633 *
1634 * Return values: see s5p_hash_final above.
1635 */
1636static int s5p_hash_digest(struct ahash_request *req)
1637{
1638	return s5p_hash_init(req) ?: s5p_hash_finup(req);
1639}
1640
1641/**
1642 * s5p_hash_cra_init_alg - init crypto alg transformation
1643 * @tfm:	crypto transformation
1644 */
1645static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm)
1646{
1647	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1648	const char *alg_name = crypto_tfm_alg_name(tfm);
1649
1650	tctx->dd = s5p_dev;
1651	/* Allocate a fallback and abort if it failed. */
1652	tctx->fallback = crypto_alloc_shash(alg_name, 0,
1653					    CRYPTO_ALG_NEED_FALLBACK);
1654	if (IS_ERR(tctx->fallback)) {
1655		pr_err("fallback alloc fails for '%s'\n", alg_name);
1656		return PTR_ERR(tctx->fallback);
1657	}
1658
1659	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1660				 sizeof(struct s5p_hash_reqctx) + BUFLEN);
1661
1662	return 0;
1663}
1664
1665/**
1666 * s5p_hash_cra_init - init crypto tfm
1667 * @tfm:	crypto transformation
1668 */
1669static int s5p_hash_cra_init(struct crypto_tfm *tfm)
1670{
1671	return s5p_hash_cra_init_alg(tfm);
1672}
1673
1674/**
1675 * s5p_hash_cra_exit - exit crypto tfm
1676 * @tfm:	crypto transformation
1677 *
1678 * free allocated fallback
1679 */
1680static void s5p_hash_cra_exit(struct crypto_tfm *tfm)
1681{
1682	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1683
1684	crypto_free_shash(tctx->fallback);
1685	tctx->fallback = NULL;
1686}
1687
1688/**
1689 * s5p_hash_export - export hash state
1690 * @req:	AHASH request
1691 * @out:	buffer for exported state
1692 */
1693static int s5p_hash_export(struct ahash_request *req, void *out)
1694{
1695	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1696
1697	memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt);
1698
1699	return 0;
1700}
1701
1702/**
1703 * s5p_hash_import - import hash state
1704 * @req:	AHASH request
1705 * @in:		buffer with state to be imported from
1706 */
1707static int s5p_hash_import(struct ahash_request *req, const void *in)
1708{
1709	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1710	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1711	struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1712	const struct s5p_hash_reqctx *ctx_in = in;
1713
1714	memcpy(ctx, in, sizeof(*ctx) + BUFLEN);
1715	if (ctx_in->bufcnt > BUFLEN) {
1716		ctx->error = true;
1717		return -EINVAL;
1718	}
1719
1720	ctx->dd = tctx->dd;
1721	ctx->error = false;
1722
1723	return 0;
1724}
1725
1726static struct ahash_alg algs_sha1_md5_sha256[] = {
1727{
1728	.init		= s5p_hash_init,
1729	.update		= s5p_hash_update,
1730	.final		= s5p_hash_final,
1731	.finup		= s5p_hash_finup,
1732	.digest		= s5p_hash_digest,
1733	.export		= s5p_hash_export,
1734	.import		= s5p_hash_import,
1735	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1736	.halg.digestsize	= SHA1_DIGEST_SIZE,
1737	.halg.base	= {
1738		.cra_name		= "sha1",
1739		.cra_driver_name	= "exynos-sha1",
1740		.cra_priority		= 100,
1741		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1742					  CRYPTO_ALG_ASYNC |
1743					  CRYPTO_ALG_NEED_FALLBACK,
1744		.cra_blocksize		= HASH_BLOCK_SIZE,
1745		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
 
1746		.cra_module		= THIS_MODULE,
1747		.cra_init		= s5p_hash_cra_init,
1748		.cra_exit		= s5p_hash_cra_exit,
1749	}
1750},
1751{
1752	.init		= s5p_hash_init,
1753	.update		= s5p_hash_update,
1754	.final		= s5p_hash_final,
1755	.finup		= s5p_hash_finup,
1756	.digest		= s5p_hash_digest,
1757	.export		= s5p_hash_export,
1758	.import		= s5p_hash_import,
1759	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1760	.halg.digestsize	= MD5_DIGEST_SIZE,
1761	.halg.base	= {
1762		.cra_name		= "md5",
1763		.cra_driver_name	= "exynos-md5",
1764		.cra_priority		= 100,
1765		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1766					  CRYPTO_ALG_ASYNC |
1767					  CRYPTO_ALG_NEED_FALLBACK,
1768		.cra_blocksize		= HASH_BLOCK_SIZE,
1769		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
 
1770		.cra_module		= THIS_MODULE,
1771		.cra_init		= s5p_hash_cra_init,
1772		.cra_exit		= s5p_hash_cra_exit,
1773	}
1774},
1775{
1776	.init		= s5p_hash_init,
1777	.update		= s5p_hash_update,
1778	.final		= s5p_hash_final,
1779	.finup		= s5p_hash_finup,
1780	.digest		= s5p_hash_digest,
1781	.export		= s5p_hash_export,
1782	.import		= s5p_hash_import,
1783	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1784	.halg.digestsize	= SHA256_DIGEST_SIZE,
1785	.halg.base	= {
1786		.cra_name		= "sha256",
1787		.cra_driver_name	= "exynos-sha256",
1788		.cra_priority		= 100,
1789		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1790					  CRYPTO_ALG_ASYNC |
1791					  CRYPTO_ALG_NEED_FALLBACK,
1792		.cra_blocksize		= HASH_BLOCK_SIZE,
1793		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
 
1794		.cra_module		= THIS_MODULE,
1795		.cra_init		= s5p_hash_cra_init,
1796		.cra_exit		= s5p_hash_cra_exit,
1797	}
1798}
1799
1800};
1801
1802static void s5p_set_aes(struct s5p_aes_dev *dev,
1803			const u8 *key, const u8 *iv, const u8 *ctr,
1804			unsigned int keylen)
1805{
1806	void __iomem *keystart;
1807
1808	if (iv)
1809		memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv,
1810			    AES_BLOCK_SIZE);
1811
1812	if (ctr)
1813		memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr,
1814			    AES_BLOCK_SIZE);
1815
1816	if (keylen == AES_KEYSIZE_256)
1817		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);
1818	else if (keylen == AES_KEYSIZE_192)
1819		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2);
1820	else
1821		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4);
1822
1823	memcpy_toio(keystart, key, keylen);
1824}
1825
1826static bool s5p_is_sg_aligned(struct scatterlist *sg)
1827{
1828	while (sg) {
1829		if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
1830			return false;
1831		sg = sg_next(sg);
1832	}
1833
1834	return true;
1835}
1836
1837static int s5p_set_indata_start(struct s5p_aes_dev *dev,
1838				struct skcipher_request *req)
1839{
1840	struct scatterlist *sg;
1841	int err;
1842
1843	dev->sg_src_cpy = NULL;
1844	sg = req->src;
1845	if (!s5p_is_sg_aligned(sg)) {
1846		dev_dbg(dev->dev,
1847			"At least one unaligned source scatter list, making a copy\n");
1848		err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy);
1849		if (err)
1850			return err;
1851
1852		sg = dev->sg_src_cpy;
1853	}
1854
1855	err = s5p_set_indata(dev, sg);
1856	if (err) {
1857		s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
1858		return err;
1859	}
1860
1861	return 0;
1862}
1863
1864static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
1865				 struct skcipher_request *req)
1866{
1867	struct scatterlist *sg;
1868	int err;
1869
1870	dev->sg_dst_cpy = NULL;
1871	sg = req->dst;
1872	if (!s5p_is_sg_aligned(sg)) {
1873		dev_dbg(dev->dev,
1874			"At least one unaligned dest scatter list, making a copy\n");
1875		err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy);
1876		if (err)
1877			return err;
1878
1879		sg = dev->sg_dst_cpy;
1880	}
1881
1882	err = s5p_set_outdata(dev, sg);
1883	if (err) {
1884		s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
1885		return err;
1886	}
1887
1888	return 0;
1889}
1890
1891static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
1892{
1893	struct skcipher_request *req = dev->req;
1894	u32 aes_control;
1895	unsigned long flags;
1896	int err;
1897	u8 *iv, *ctr;
1898
1899	/* This sets bit [13:12] to 00, which selects 128-bit counter */
1900	aes_control = SSS_AES_KEY_CHANGE_MODE;
1901	if (mode & FLAGS_AES_DECRYPT)
1902		aes_control |= SSS_AES_MODE_DECRYPT;
1903
1904	if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
1905		aes_control |= SSS_AES_CHAIN_MODE_CBC;
1906		iv = req->iv;
1907		ctr = NULL;
1908	} else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
1909		aes_control |= SSS_AES_CHAIN_MODE_CTR;
1910		iv = NULL;
1911		ctr = req->iv;
1912	} else {
1913		iv = NULL; /* AES_ECB */
1914		ctr = NULL;
1915	}
1916
1917	if (dev->ctx->keylen == AES_KEYSIZE_192)
1918		aes_control |= SSS_AES_KEY_SIZE_192;
1919	else if (dev->ctx->keylen == AES_KEYSIZE_256)
1920		aes_control |= SSS_AES_KEY_SIZE_256;
1921
1922	aes_control |= SSS_AES_FIFO_MODE;
1923
1924	/* as a variant it is possible to use byte swapping on DMA side */
1925	aes_control |= SSS_AES_BYTESWAP_DI
1926		    |  SSS_AES_BYTESWAP_DO
1927		    |  SSS_AES_BYTESWAP_IV
1928		    |  SSS_AES_BYTESWAP_KEY
1929		    |  SSS_AES_BYTESWAP_CNT;
1930
1931	spin_lock_irqsave(&dev->lock, flags);
1932
1933	SSS_WRITE(dev, FCINTENCLR,
1934		  SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);
1935	SSS_WRITE(dev, FCFIFOCTRL, 0x00);
1936
1937	err = s5p_set_indata_start(dev, req);
1938	if (err)
1939		goto indata_error;
1940
1941	err = s5p_set_outdata_start(dev, req);
1942	if (err)
1943		goto outdata_error;
1944
1945	SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
1946	s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen);
1947
1948	s5p_set_dma_indata(dev,  dev->sg_src);
1949	s5p_set_dma_outdata(dev, dev->sg_dst);
1950
1951	SSS_WRITE(dev, FCINTENSET,
1952		  SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);
1953
1954	spin_unlock_irqrestore(&dev->lock, flags);
1955
1956	return;
1957
1958outdata_error:
1959	s5p_unset_indata(dev);
1960
1961indata_error:
1962	s5p_sg_done(dev);
1963	dev->busy = false;
1964	spin_unlock_irqrestore(&dev->lock, flags);
1965	s5p_aes_complete(req, err);
1966}
1967
1968static void s5p_tasklet_cb(unsigned long data)
1969{
1970	struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;
1971	struct crypto_async_request *async_req, *backlog;
1972	struct s5p_aes_reqctx *reqctx;
1973	unsigned long flags;
1974
1975	spin_lock_irqsave(&dev->lock, flags);
1976	backlog   = crypto_get_backlog(&dev->queue);
1977	async_req = crypto_dequeue_request(&dev->queue);
1978
1979	if (!async_req) {
1980		dev->busy = false;
1981		spin_unlock_irqrestore(&dev->lock, flags);
1982		return;
1983	}
1984	spin_unlock_irqrestore(&dev->lock, flags);
1985
1986	if (backlog)
1987		crypto_request_complete(backlog, -EINPROGRESS);
1988
1989	dev->req = skcipher_request_cast(async_req);
1990	dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);
1991	reqctx   = skcipher_request_ctx(dev->req);
1992
1993	s5p_aes_crypt_start(dev, reqctx->mode);
1994}
1995
1996static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
1997			      struct skcipher_request *req)
1998{
1999	unsigned long flags;
2000	int err;
2001
2002	spin_lock_irqsave(&dev->lock, flags);
2003	err = crypto_enqueue_request(&dev->queue, &req->base);
2004	if (dev->busy) {
2005		spin_unlock_irqrestore(&dev->lock, flags);
2006		return err;
2007	}
2008	dev->busy = true;
2009
2010	spin_unlock_irqrestore(&dev->lock, flags);
2011
2012	tasklet_schedule(&dev->tasklet);
2013
2014	return err;
2015}
2016
2017static int s5p_aes_crypt(struct skcipher_request *req, unsigned long mode)
2018{
2019	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2020	struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req);
2021	struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
2022	struct s5p_aes_dev *dev = ctx->dev;
2023
2024	if (!req->cryptlen)
2025		return 0;
2026
2027	if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE) &&
2028			((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) {
2029		dev_dbg(dev->dev, "request size is not exact amount of AES blocks\n");
2030		return -EINVAL;
2031	}
2032
2033	reqctx->mode = mode;
2034
2035	return s5p_aes_handle_req(dev, req);
2036}
2037
2038static int s5p_aes_setkey(struct crypto_skcipher *cipher,
2039			  const u8 *key, unsigned int keylen)
2040{
2041	struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
2042	struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
2043
2044	if (keylen != AES_KEYSIZE_128 &&
2045	    keylen != AES_KEYSIZE_192 &&
2046	    keylen != AES_KEYSIZE_256)
2047		return -EINVAL;
2048
2049	memcpy(ctx->aes_key, key, keylen);
2050	ctx->keylen = keylen;
2051
2052	return 0;
2053}
2054
2055static int s5p_aes_ecb_encrypt(struct skcipher_request *req)
2056{
2057	return s5p_aes_crypt(req, 0);
2058}
2059
2060static int s5p_aes_ecb_decrypt(struct skcipher_request *req)
2061{
2062	return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);
2063}
2064
2065static int s5p_aes_cbc_encrypt(struct skcipher_request *req)
2066{
2067	return s5p_aes_crypt(req, FLAGS_AES_CBC);
2068}
2069
2070static int s5p_aes_cbc_decrypt(struct skcipher_request *req)
2071{
2072	return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
2073}
2074
2075static int s5p_aes_ctr_crypt(struct skcipher_request *req)
2076{
2077	return s5p_aes_crypt(req, FLAGS_AES_CTR);
2078}
2079
2080static int s5p_aes_init_tfm(struct crypto_skcipher *tfm)
2081{
2082	struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
2083
2084	ctx->dev = s5p_dev;
2085	crypto_skcipher_set_reqsize(tfm, sizeof(struct s5p_aes_reqctx));
2086
2087	return 0;
2088}
2089
2090static struct skcipher_alg algs[] = {
2091	{
2092		.base.cra_name		= "ecb(aes)",
2093		.base.cra_driver_name	= "ecb-aes-s5p",
2094		.base.cra_priority	= 100,
2095		.base.cra_flags		= CRYPTO_ALG_ASYNC |
2096					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2097		.base.cra_blocksize	= AES_BLOCK_SIZE,
2098		.base.cra_ctxsize	= sizeof(struct s5p_aes_ctx),
2099		.base.cra_alignmask	= 0x0f,
2100		.base.cra_module	= THIS_MODULE,
2101
2102		.min_keysize		= AES_MIN_KEY_SIZE,
2103		.max_keysize		= AES_MAX_KEY_SIZE,
2104		.setkey			= s5p_aes_setkey,
2105		.encrypt		= s5p_aes_ecb_encrypt,
2106		.decrypt		= s5p_aes_ecb_decrypt,
2107		.init			= s5p_aes_init_tfm,
2108	},
2109	{
2110		.base.cra_name		= "cbc(aes)",
2111		.base.cra_driver_name	= "cbc-aes-s5p",
2112		.base.cra_priority	= 100,
2113		.base.cra_flags		= CRYPTO_ALG_ASYNC |
2114					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2115		.base.cra_blocksize	= AES_BLOCK_SIZE,
2116		.base.cra_ctxsize	= sizeof(struct s5p_aes_ctx),
2117		.base.cra_alignmask	= 0x0f,
2118		.base.cra_module	= THIS_MODULE,
2119
2120		.min_keysize		= AES_MIN_KEY_SIZE,
2121		.max_keysize		= AES_MAX_KEY_SIZE,
2122		.ivsize			= AES_BLOCK_SIZE,
2123		.setkey			= s5p_aes_setkey,
2124		.encrypt		= s5p_aes_cbc_encrypt,
2125		.decrypt		= s5p_aes_cbc_decrypt,
2126		.init			= s5p_aes_init_tfm,
2127	},
2128	{
2129		.base.cra_name		= "ctr(aes)",
2130		.base.cra_driver_name	= "ctr-aes-s5p",
2131		.base.cra_priority	= 100,
2132		.base.cra_flags		= CRYPTO_ALG_ASYNC |
2133					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2134		.base.cra_blocksize	= 1,
2135		.base.cra_ctxsize	= sizeof(struct s5p_aes_ctx),
2136		.base.cra_alignmask	= 0x0f,
2137		.base.cra_module	= THIS_MODULE,
2138
2139		.min_keysize		= AES_MIN_KEY_SIZE,
2140		.max_keysize		= AES_MAX_KEY_SIZE,
2141		.ivsize			= AES_BLOCK_SIZE,
2142		.setkey			= s5p_aes_setkey,
2143		.encrypt		= s5p_aes_ctr_crypt,
2144		.decrypt		= s5p_aes_ctr_crypt,
2145		.init			= s5p_aes_init_tfm,
2146	},
2147};
2148
2149static int s5p_aes_probe(struct platform_device *pdev)
2150{
2151	struct device *dev = &pdev->dev;
2152	int i, j, err;
2153	const struct samsung_aes_variant *variant;
2154	struct s5p_aes_dev *pdata;
2155	struct resource *res;
2156	unsigned int hash_i;
2157
2158	if (s5p_dev)
2159		return -EEXIST;
2160
2161	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
2162	if (!pdata)
2163		return -ENOMEM;
2164
2165	variant = find_s5p_sss_version(pdev);
2166	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2167	if (!res)
2168		return -EINVAL;
2169
2170	/*
2171	 * Note: HASH and PRNG uses the same registers in secss, avoid
2172	 * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG
2173	 * is enabled in config. We need larger size for HASH registers in
2174	 * secss, current describe only AES/DES
2175	 */
2176	if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) {
2177		if (variant == &exynos_aes_data) {
2178			res->end += 0x300;
2179			pdata->use_hash = true;
2180		}
2181	}
2182
2183	pdata->res = res;
2184	pdata->ioaddr = devm_ioremap_resource(dev, res);
2185	if (IS_ERR(pdata->ioaddr)) {
2186		if (!pdata->use_hash)
2187			return PTR_ERR(pdata->ioaddr);
2188		/* try AES without HASH */
2189		res->end -= 0x300;
2190		pdata->use_hash = false;
2191		pdata->ioaddr = devm_ioremap_resource(dev, res);
2192		if (IS_ERR(pdata->ioaddr))
2193			return PTR_ERR(pdata->ioaddr);
2194	}
2195
2196	pdata->clk = devm_clk_get(dev, variant->clk_names[0]);
2197	if (IS_ERR(pdata->clk))
2198		return dev_err_probe(dev, PTR_ERR(pdata->clk),
2199				     "failed to find secss clock %s\n",
2200				     variant->clk_names[0]);
2201
2202	err = clk_prepare_enable(pdata->clk);
2203	if (err < 0) {
2204		dev_err(dev, "Enabling clock %s failed, err %d\n",
2205			variant->clk_names[0], err);
2206		return err;
2207	}
2208
2209	if (variant->clk_names[1]) {
2210		pdata->pclk = devm_clk_get(dev, variant->clk_names[1]);
2211		if (IS_ERR(pdata->pclk)) {
2212			err = dev_err_probe(dev, PTR_ERR(pdata->pclk),
2213					    "failed to find clock %s\n",
2214					    variant->clk_names[1]);
2215			goto err_clk;
2216		}
2217
2218		err = clk_prepare_enable(pdata->pclk);
2219		if (err < 0) {
2220			dev_err(dev, "Enabling clock %s failed, err %d\n",
2221				variant->clk_names[0], err);
2222			goto err_clk;
2223		}
2224	} else {
2225		pdata->pclk = NULL;
2226	}
2227
2228	spin_lock_init(&pdata->lock);
2229	spin_lock_init(&pdata->hash_lock);
2230
2231	pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset;
2232	pdata->io_hash_base = pdata->ioaddr + variant->hash_offset;
2233
2234	pdata->irq_fc = platform_get_irq(pdev, 0);
2235	if (pdata->irq_fc < 0) {
2236		err = pdata->irq_fc;
2237		dev_warn(dev, "feed control interrupt is not available.\n");
2238		goto err_irq;
2239	}
2240	err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,
2241					s5p_aes_interrupt, IRQF_ONESHOT,
2242					pdev->name, pdev);
2243	if (err < 0) {
2244		dev_warn(dev, "feed control interrupt is not available.\n");
2245		goto err_irq;
2246	}
2247
2248	pdata->busy = false;
2249	pdata->dev = dev;
2250	platform_set_drvdata(pdev, pdata);
2251	s5p_dev = pdata;
2252
2253	tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);
2254	crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
2255
2256	for (i = 0; i < ARRAY_SIZE(algs); i++) {
2257		err = crypto_register_skcipher(&algs[i]);
2258		if (err)
2259			goto err_algs;
2260	}
2261
2262	if (pdata->use_hash) {
2263		tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb,
2264			     (unsigned long)pdata);
2265		crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH);
2266
2267		for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256);
2268		     hash_i++) {
2269			struct ahash_alg *alg;
2270
2271			alg = &algs_sha1_md5_sha256[hash_i];
2272			err = crypto_register_ahash(alg);
2273			if (err) {
2274				dev_err(dev, "can't register '%s': %d\n",
2275					alg->halg.base.cra_driver_name, err);
2276				goto err_hash;
2277			}
2278		}
2279	}
2280
2281	dev_info(dev, "s5p-sss driver registered\n");
2282
2283	return 0;
2284
2285err_hash:
2286	for (j = hash_i - 1; j >= 0; j--)
2287		crypto_unregister_ahash(&algs_sha1_md5_sha256[j]);
2288
2289	tasklet_kill(&pdata->hash_tasklet);
2290	res->end -= 0x300;
2291
2292err_algs:
2293	if (i < ARRAY_SIZE(algs))
2294		dev_err(dev, "can't register '%s': %d\n", algs[i].base.cra_name,
2295			err);
2296
2297	for (j = 0; j < i; j++)
2298		crypto_unregister_skcipher(&algs[j]);
2299
2300	tasklet_kill(&pdata->tasklet);
2301
2302err_irq:
2303	clk_disable_unprepare(pdata->pclk);
2304
2305err_clk:
2306	clk_disable_unprepare(pdata->clk);
2307	s5p_dev = NULL;
2308
2309	return err;
2310}
2311
2312static void s5p_aes_remove(struct platform_device *pdev)
2313{
2314	struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);
2315	int i;
2316
2317	for (i = 0; i < ARRAY_SIZE(algs); i++)
2318		crypto_unregister_skcipher(&algs[i]);
2319
2320	tasklet_kill(&pdata->tasklet);
2321	if (pdata->use_hash) {
2322		for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--)
2323			crypto_unregister_ahash(&algs_sha1_md5_sha256[i]);
2324
2325		pdata->res->end -= 0x300;
2326		tasklet_kill(&pdata->hash_tasklet);
2327		pdata->use_hash = false;
2328	}
2329
2330	clk_disable_unprepare(pdata->pclk);
2331
2332	clk_disable_unprepare(pdata->clk);
2333	s5p_dev = NULL;
 
 
2334}
2335
2336static struct platform_driver s5p_aes_crypto = {
2337	.probe	= s5p_aes_probe,
2338	.remove_new = s5p_aes_remove,
2339	.driver	= {
2340		.name	= "s5p-secss",
2341		.of_match_table = s5p_sss_dt_match,
2342	},
2343};
2344
2345module_platform_driver(s5p_aes_crypto);
2346
2347MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
2348MODULE_LICENSE("GPL v2");
2349MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");
2350MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>");