1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * This file is part of STM32 Crypto driver for Linux.
   4 *
   5 * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
   6 * Author(s): Lionel DEBIEVE <lionel.debieve@st.com> for STMicroelectronics.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
   7 */
   8
   9#include <crypto/engine.h>
  10#include <crypto/internal/hash.h>
  11#include <crypto/md5.h>
  12#include <crypto/scatterwalk.h>
  13#include <crypto/sha1.h>
  14#include <crypto/sha2.h>
  15#include <crypto/sha3.h>
  16#include <linux/clk.h>
 
  17#include <linux/delay.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/dmaengine.h>
  20#include <linux/interrupt.h>
 
  21#include <linux/iopoll.h>
  22#include <linux/kernel.h>
  23#include <linux/module.h>
  24#include <linux/of.h>
  25#include <linux/platform_device.h>
  26#include <linux/pm_runtime.h>
  27#include <linux/reset.h>
  28#include <linux/string.h>
 
 
 
 
 
 
  29
  30#define HASH_CR				0x00
  31#define HASH_DIN			0x04
  32#define HASH_STR			0x08
  33#define HASH_UX500_HREG(x)		(0x0c + ((x) * 0x04))
  34#define HASH_IMR			0x20
  35#define HASH_SR				0x24
  36#define HASH_CSR(x)			(0x0F8 + ((x) * 0x04))
  37#define HASH_HREG(x)			(0x310 + ((x) * 0x04))
  38#define HASH_HWCFGR			0x3F0
  39#define HASH_VER			0x3F4
  40#define HASH_ID				0x3F8
  41
  42/* Control Register */
  43#define HASH_CR_INIT			BIT(2)
  44#define HASH_CR_DMAE			BIT(3)
  45#define HASH_CR_DATATYPE_POS		4
  46#define HASH_CR_MODE			BIT(6)
  47#define HASH_CR_ALGO_POS		7
  48#define HASH_CR_MDMAT			BIT(13)
  49#define HASH_CR_DMAA			BIT(14)
  50#define HASH_CR_LKEY			BIT(16)
  51
 
 
 
 
 
  52/* Interrupt */
  53#define HASH_DINIE			BIT(0)
  54#define HASH_DCIE			BIT(1)
  55
  56/* Interrupt Mask */
  57#define HASH_MASK_CALC_COMPLETION	BIT(0)
  58#define HASH_MASK_DATA_INPUT		BIT(1)
  59
 
 
 
  60/* Status Flags */
  61#define HASH_SR_DATA_INPUT_READY	BIT(0)
  62#define HASH_SR_OUTPUT_READY		BIT(1)
  63#define HASH_SR_DMA_ACTIVE		BIT(2)
  64#define HASH_SR_BUSY			BIT(3)
  65
  66/* STR Register */
  67#define HASH_STR_NBLW_MASK		GENMASK(4, 0)
  68#define HASH_STR_DCAL			BIT(8)
  69
  70/* HWCFGR Register */
  71#define HASH_HWCFG_DMA_MASK		GENMASK(3, 0)
  72
  73/* Context swap register */
  74#define HASH_CSR_NB_SHA256_HMAC		54
  75#define HASH_CSR_NB_SHA256		38
  76#define HASH_CSR_NB_SHA512_HMAC		103
  77#define HASH_CSR_NB_SHA512		91
  78#define HASH_CSR_NB_SHA3_HMAC		88
  79#define HASH_CSR_NB_SHA3		72
  80#define HASH_CSR_NB_MAX			HASH_CSR_NB_SHA512_HMAC
  81
  82#define HASH_FLAGS_INIT			BIT(0)
  83#define HASH_FLAGS_OUTPUT_READY		BIT(1)
  84#define HASH_FLAGS_CPU			BIT(2)
  85#define HASH_FLAGS_DMA_ACTIVE		BIT(3)
  86#define HASH_FLAGS_HMAC_INIT		BIT(4)
  87#define HASH_FLAGS_HMAC_FINAL		BIT(5)
  88#define HASH_FLAGS_HMAC_KEY		BIT(6)
  89#define HASH_FLAGS_SHA3_MODE		BIT(7)
 
  90#define HASH_FLAGS_FINAL		BIT(15)
  91#define HASH_FLAGS_FINUP		BIT(16)
  92#define HASH_FLAGS_ALGO_MASK		GENMASK(20, 17)
  93#define HASH_FLAGS_ALGO_SHIFT		17
  94#define HASH_FLAGS_ERRORS		BIT(21)
  95#define HASH_FLAGS_EMPTY		BIT(22)
 
 
  96#define HASH_FLAGS_HMAC			BIT(23)
  97#define HASH_FLAGS_SGS_COPIED		BIT(24)
  98
  99#define HASH_OP_UPDATE			1
 100#define HASH_OP_FINAL			2
 101
 102#define HASH_BURST_LEVEL		4
 103
 104enum stm32_hash_data_format {
 105	HASH_DATA_32_BITS		= 0x0,
 106	HASH_DATA_16_BITS		= 0x1,
 107	HASH_DATA_8_BITS		= 0x2,
 108	HASH_DATA_1_BIT			= 0x3
 109};
 110
 111#define HASH_BUFLEN			(SHA3_224_BLOCK_SIZE + 4)
 112#define HASH_MAX_KEY_SIZE		(SHA512_BLOCK_SIZE * 8)
 113
 114enum stm32_hash_algo {
 115	HASH_SHA1			= 0,
 116	HASH_MD5			= 1,
 117	HASH_SHA224			= 2,
 118	HASH_SHA256			= 3,
 119	HASH_SHA3_224			= 4,
 120	HASH_SHA3_256			= 5,
 121	HASH_SHA3_384			= 6,
 122	HASH_SHA3_512			= 7,
 123	HASH_SHA384			= 12,
 124	HASH_SHA512			= 15,
 125};
 126
 127enum ux500_hash_algo {
 128	HASH_SHA256_UX500		= 0,
 129	HASH_SHA1_UX500			= 1,
 130};
 131
 132#define HASH_AUTOSUSPEND_DELAY		50
 133
 134struct stm32_hash_ctx {
 
 135	struct stm32_hash_dev	*hdev;
 136	struct crypto_shash	*xtfm;
 137	unsigned long		flags;
 138
 139	u8			key[HASH_MAX_KEY_SIZE];
 140	int			keylen;
 141};
 142
 143struct stm32_hash_state {
 144	u32			flags;
 145
 146	u16			bufcnt;
 147	u16			blocklen;
 148
 149	u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32));
 150
 151	/* hash state */
 152	u32			hw_context[3 + HASH_CSR_NB_MAX];
 153};
 154
 155struct stm32_hash_request_ctx {
 156	struct stm32_hash_dev	*hdev;
 
 157	unsigned long		op;
 158
 159	u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
 160	size_t			digcnt;
 
 
 161
 
 162	struct scatterlist	*sg;
 163	struct scatterlist	sgl[2]; /* scatterlist used to realize alignment */
 164	unsigned int		offset;
 165	unsigned int		total;
 166	struct scatterlist	sg_key;
 167
 168	dma_addr_t		dma_addr;
 169	size_t			dma_ct;
 170	int			nents;
 171
 172	u8			data_type;
 173
 174	struct stm32_hash_state state;
 
 
 
 175};
 176
 177struct stm32_hash_algs_info {
 178	struct ahash_engine_alg	*algs_list;
 179	size_t			size;
 180};
 181
 182struct stm32_hash_pdata {
 183	const int				alg_shift;
 184	const struct stm32_hash_algs_info	*algs_info;
 185	size_t					algs_info_size;
 186	bool					has_sr;
 187	bool					has_mdmat;
 188	bool					context_secured;
 189	bool					broken_emptymsg;
 190	bool					ux500;
 191};
 192
 193struct stm32_hash_dev {
 194	struct list_head	list;
 195	struct device		*dev;
 196	struct clk		*clk;
 197	struct reset_control	*rst;
 198	void __iomem		*io_base;
 199	phys_addr_t		phys_base;
 200	u8			xmit_buf[HASH_BUFLEN] __aligned(sizeof(u32));
 201	u32			dma_mode;
 202	bool			polled;
 
 
 203
 204	struct ahash_request	*req;
 205	struct crypto_engine	*engine;
 206
 
 207	unsigned long		flags;
 208
 209	struct dma_chan		*dma_lch;
 210	struct completion	dma_completion;
 211
 212	const struct stm32_hash_pdata	*pdata;
 213};
 214
 215struct stm32_hash_drv {
 216	struct list_head	dev_list;
 217	spinlock_t		lock; /* List protection access */
 218};
 219
 220static struct stm32_hash_drv stm32_hash = {
 221	.dev_list = LIST_HEAD_INIT(stm32_hash.dev_list),
 222	.lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock),
 223};
 224
 225static void stm32_hash_dma_callback(void *param);
 226static int stm32_hash_prepare_request(struct ahash_request *req);
 227static void stm32_hash_unprepare_request(struct ahash_request *req);
 228
 229static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset)
 230{
 231	return readl_relaxed(hdev->io_base + offset);
 232}
 233
 234static inline void stm32_hash_write(struct stm32_hash_dev *hdev,
 235				    u32 offset, u32 value)
 236{
 237	writel_relaxed(value, hdev->io_base + offset);
 238}
 239
 240/**
 241 * stm32_hash_wait_busy - wait until hash processor is available. It return an
 242 * error if the hash core is processing a block of data for more than 10 ms.
 243 * @hdev: the stm32_hash_dev device.
 244 */
 245static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev)
 246{
 247	u32 status;
 248
 249	/* The Ux500 lacks the special status register, we poll the DCAL bit instead */
 250	if (!hdev->pdata->has_sr)
 251		return readl_relaxed_poll_timeout(hdev->io_base + HASH_STR, status,
 252						  !(status & HASH_STR_DCAL), 10, 10000);
 253
 254	return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status,
 255				   !(status & HASH_SR_BUSY), 10, 10000);
 256}
 257
 258/**
 259 * stm32_hash_set_nblw - set the number of valid bytes in the last word.
 260 * @hdev: the stm32_hash_dev device.
 261 * @length: the length of the final word.
 262 */
 263static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length)
 264{
 265	u32 reg;
 266
 267	reg = stm32_hash_read(hdev, HASH_STR);
 268	reg &= ~(HASH_STR_NBLW_MASK);
 269	reg |= (8U * ((length) % 4U));
 270	stm32_hash_write(hdev, HASH_STR, reg);
 271}
 272
 273static int stm32_hash_write_key(struct stm32_hash_dev *hdev)
 274{
 275	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 276	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 277	u32 reg;
 278	int keylen = ctx->keylen;
 279	void *key = ctx->key;
 280
 281	if (keylen) {
 282		stm32_hash_set_nblw(hdev, keylen);
 283
 284		while (keylen > 0) {
 285			stm32_hash_write(hdev, HASH_DIN, *(u32 *)key);
 286			keylen -= 4;
 287			key += 4;
 288		}
 289
 290		reg = stm32_hash_read(hdev, HASH_STR);
 291		reg |= HASH_STR_DCAL;
 292		stm32_hash_write(hdev, HASH_STR, reg);
 293
 294		return -EINPROGRESS;
 295	}
 296
 297	return 0;
 298}
 299
 300/**
 301 * stm32_hash_write_ctrl - Initialize the hash processor, only if
 302 * HASH_FLAGS_INIT is set.
 303 * @hdev: the stm32_hash_dev device
 304 */
 305static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev)
 306{
 307	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 308	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 309	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 310	struct stm32_hash_state *state = &rctx->state;
 311	u32 alg = (state->flags & HASH_FLAGS_ALGO_MASK) >> HASH_FLAGS_ALGO_SHIFT;
 312
 313	u32 reg = HASH_CR_INIT;
 314
 315	if (!(hdev->flags & HASH_FLAGS_INIT)) {
 316		if (hdev->pdata->ux500) {
 317			reg |= ((alg & BIT(0)) << HASH_CR_ALGO_POS);
 318		} else {
 319			if (hdev->pdata->alg_shift == HASH_CR_ALGO_POS)
 320				reg |= ((alg & BIT(1)) << 17) |
 321				       ((alg & BIT(0)) << HASH_CR_ALGO_POS);
 322			else
 323				reg |= alg << hdev->pdata->alg_shift;
 
 
 
 
 
 
 
 324		}
 325
 326		reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
 327
 328		if (state->flags & HASH_FLAGS_HMAC) {
 329			hdev->flags |= HASH_FLAGS_HMAC;
 330			reg |= HASH_CR_MODE;
 331			if (ctx->keylen > crypto_ahash_blocksize(tfm))
 332				reg |= HASH_CR_LKEY;
 333		}
 334
 335		if (!hdev->polled)
 336			stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
 337
 338		stm32_hash_write(hdev, HASH_CR, reg);
 339
 340		hdev->flags |= HASH_FLAGS_INIT;
 341
 342		/*
 343		 * After first block + 1 words are fill up,
 344		 * we only need to fill 1 block to start partial computation
 345		 */
 346		rctx->state.blocklen -= sizeof(u32);
 347
 348		dev_dbg(hdev->dev, "Write Control %x\n", reg);
 349	}
 350}
 351
 352static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
 353{
 354	struct stm32_hash_state *state = &rctx->state;
 355	size_t count;
 356
 357	while ((state->bufcnt < state->blocklen) && rctx->total) {
 358		count = min(rctx->sg->length - rctx->offset, rctx->total);
 359		count = min_t(size_t, count, state->blocklen - state->bufcnt);
 360
 361		if (count <= 0) {
 362			if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
 363				rctx->sg = sg_next(rctx->sg);
 364				continue;
 365			} else {
 366				break;
 367			}
 368		}
 369
 370		scatterwalk_map_and_copy(state->buffer + state->bufcnt,
 371					 rctx->sg, rctx->offset, count, 0);
 372
 373		state->bufcnt += count;
 374		rctx->offset += count;
 375		rctx->total -= count;
 376
 377		if (rctx->offset == rctx->sg->length) {
 378			rctx->sg = sg_next(rctx->sg);
 379			if (rctx->sg)
 380				rctx->offset = 0;
 381			else
 382				rctx->total = 0;
 383		}
 384	}
 385}
 386
 387static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
 388			       const u8 *buf, size_t length, int final)
 389{
 390	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 391	struct stm32_hash_state *state = &rctx->state;
 392	unsigned int count, len32;
 393	const u32 *buffer = (const u32 *)buf;
 394	u32 reg;
 395
 396	if (final) {
 397		hdev->flags |= HASH_FLAGS_FINAL;
 398
 399		/* Do not process empty messages if hw is buggy. */
 400		if (!(hdev->flags & HASH_FLAGS_INIT) && !length &&
 401		    hdev->pdata->broken_emptymsg) {
 402			state->flags |= HASH_FLAGS_EMPTY;
 403			return 0;
 404		}
 405	}
 406
 407	len32 = DIV_ROUND_UP(length, sizeof(u32));
 408
 409	dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n",
 410		__func__, length, final, len32);
 411
 412	hdev->flags |= HASH_FLAGS_CPU;
 413
 414	stm32_hash_write_ctrl(hdev);
 415
 416	if (stm32_hash_wait_busy(hdev))
 417		return -ETIMEDOUT;
 418
 419	if ((hdev->flags & HASH_FLAGS_HMAC) &&
 420	    (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
 421		hdev->flags |= HASH_FLAGS_HMAC_KEY;
 422		stm32_hash_write_key(hdev);
 423		if (stm32_hash_wait_busy(hdev))
 424			return -ETIMEDOUT;
 425	}
 426
 427	for (count = 0; count < len32; count++)
 428		stm32_hash_write(hdev, HASH_DIN, buffer[count]);
 429
 430	if (final) {
 431		if (stm32_hash_wait_busy(hdev))
 432			return -ETIMEDOUT;
 433
 434		stm32_hash_set_nblw(hdev, length);
 435		reg = stm32_hash_read(hdev, HASH_STR);
 436		reg |= HASH_STR_DCAL;
 437		stm32_hash_write(hdev, HASH_STR, reg);
 438		if (hdev->flags & HASH_FLAGS_HMAC) {
 439			if (stm32_hash_wait_busy(hdev))
 440				return -ETIMEDOUT;
 441			stm32_hash_write_key(hdev);
 442		}
 443		return -EINPROGRESS;
 444	}
 445
 446	return 0;
 447}
 448
 449static int hash_swap_reg(struct stm32_hash_request_ctx *rctx)
 450{
 451	struct stm32_hash_state *state = &rctx->state;
 452
 453	switch ((state->flags & HASH_FLAGS_ALGO_MASK) >>
 454		HASH_FLAGS_ALGO_SHIFT) {
 455	case HASH_MD5:
 456	case HASH_SHA1:
 457	case HASH_SHA224:
 458	case HASH_SHA256:
 459		if (state->flags & HASH_FLAGS_HMAC)
 460			return HASH_CSR_NB_SHA256_HMAC;
 461		else
 462			return HASH_CSR_NB_SHA256;
 463		break;
 464
 465	case HASH_SHA384:
 466	case HASH_SHA512:
 467		if (state->flags & HASH_FLAGS_HMAC)
 468			return HASH_CSR_NB_SHA512_HMAC;
 469		else
 470			return HASH_CSR_NB_SHA512;
 471		break;
 472
 473	case HASH_SHA3_224:
 474	case HASH_SHA3_256:
 475	case HASH_SHA3_384:
 476	case HASH_SHA3_512:
 477		if (state->flags & HASH_FLAGS_HMAC)
 478			return HASH_CSR_NB_SHA3_HMAC;
 479		else
 480			return HASH_CSR_NB_SHA3;
 481		break;
 482
 483	default:
 484		return -EINVAL;
 485	}
 486}
 487
 488static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
 489{
 490	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 491	struct stm32_hash_state *state = &rctx->state;
 492	int bufcnt, err = 0, final;
 493
 494	dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags);
 495
 496	final = state->flags & HASH_FLAGS_FINAL;
 497
 498	while ((rctx->total >= state->blocklen) ||
 499	       (state->bufcnt + rctx->total >= state->blocklen)) {
 500		stm32_hash_append_sg(rctx);
 501		bufcnt = state->bufcnt;
 502		state->bufcnt = 0;
 503		err = stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 0);
 504		if (err)
 505			return err;
 506	}
 507
 508	stm32_hash_append_sg(rctx);
 509
 510	if (final) {
 511		bufcnt = state->bufcnt;
 512		state->bufcnt = 0;
 513		return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1);
 
 514	}
 515
 516	return err;
 517}
 518
 519static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
 520			       struct scatterlist *sg, int length, int mdmat)
 521{
 522	struct dma_async_tx_descriptor *in_desc;
 523	dma_cookie_t cookie;
 524	u32 reg;
 525	int err;
 526
 527	dev_dbg(hdev->dev, "%s mdmat: %x length: %d\n", __func__, mdmat, length);
 528
 529	/* do not use dma if there is no data to send */
 530	if (length <= 0)
 531		return 0;
 532
 533	in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1,
 534					  DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT |
 535					  DMA_CTRL_ACK);
 536	if (!in_desc) {
 537		dev_err(hdev->dev, "dmaengine_prep_slave error\n");
 538		return -ENOMEM;
 539	}
 540
 541	reinit_completion(&hdev->dma_completion);
 542	in_desc->callback = stm32_hash_dma_callback;
 543	in_desc->callback_param = hdev;
 544
 
 545	hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
 546
 547	reg = stm32_hash_read(hdev, HASH_CR);
 548
 549	if (hdev->pdata->has_mdmat) {
 550		if (mdmat)
 551			reg |= HASH_CR_MDMAT;
 552		else
 553			reg &= ~HASH_CR_MDMAT;
 554	}
 555	reg |= HASH_CR_DMAE;
 556
 557	stm32_hash_write(hdev, HASH_CR, reg);
 558
 
 559
 560	cookie = dmaengine_submit(in_desc);
 561	err = dma_submit_error(cookie);
 562	if (err)
 563		return -ENOMEM;
 564
 565	dma_async_issue_pending(hdev->dma_lch);
 566
 567	if (!wait_for_completion_timeout(&hdev->dma_completion,
 568					 msecs_to_jiffies(100)))
 569		err = -ETIMEDOUT;
 570
 571	if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
 572				     NULL, NULL) != DMA_COMPLETE)
 573		err = -ETIMEDOUT;
 574
 575	if (err) {
 576		dev_err(hdev->dev, "DMA Error %i\n", err);
 577		dmaengine_terminate_all(hdev->dma_lch);
 578		return err;
 579	}
 580
 581	return -EINPROGRESS;
 582}
 583
 584static void stm32_hash_dma_callback(void *param)
 585{
 586	struct stm32_hash_dev *hdev = param;
 587
 588	complete(&hdev->dma_completion);
 
 
 589}
 590
 591static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
 592{
 593	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 594	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 595	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 596	int err;
 597
 598	if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode > 0) {
 599		err = stm32_hash_write_key(hdev);
 600		if (stm32_hash_wait_busy(hdev))
 601			return -ETIMEDOUT;
 602	} else {
 603		if (!(hdev->flags & HASH_FLAGS_HMAC_KEY))
 604			sg_init_one(&rctx->sg_key, ctx->key,
 605				    ALIGN(ctx->keylen, sizeof(u32)));
 606
 607		rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1,
 608					  DMA_TO_DEVICE);
 609		if (rctx->dma_ct == 0) {
 610			dev_err(hdev->dev, "dma_map_sg error\n");
 611			return -ENOMEM;
 612		}
 613
 614		err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0);
 615
 616		dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE);
 617	}
 618
 619	return err;
 620}
 621
 622static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
 623{
 624	struct dma_slave_config dma_conf;
 625	struct dma_chan *chan;
 626	int err;
 627
 628	memset(&dma_conf, 0, sizeof(dma_conf));
 629
 630	dma_conf.direction = DMA_MEM_TO_DEV;
 631	dma_conf.dst_addr = hdev->phys_base + HASH_DIN;
 632	dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 633	dma_conf.src_maxburst = HASH_BURST_LEVEL;
 634	dma_conf.dst_maxburst = HASH_BURST_LEVEL;
 635	dma_conf.device_fc = false;
 636
 637	chan = dma_request_chan(hdev->dev, "in");
 638	if (IS_ERR(chan))
 639		return PTR_ERR(chan);
 640
 641	hdev->dma_lch = chan;
 642
 643	err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
 644	if (err) {
 645		dma_release_channel(hdev->dma_lch);
 646		hdev->dma_lch = NULL;
 647		dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
 648		return err;
 649	}
 650
 651	init_completion(&hdev->dma_completion);
 652
 653	return 0;
 654}
 655
 656static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
 657{
 658	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 659	u32 *buffer = (void *)rctx->state.buffer;
 660	struct scatterlist sg[1], *tsg;
 661	int err = 0, reg, ncp = 0;
 662	unsigned int i, len = 0, bufcnt = 0;
 663	bool final = hdev->flags & HASH_FLAGS_FINAL;
 664	bool is_last = false;
 665	u32 last_word;
 666
 667	dev_dbg(hdev->dev, "%s total: %d bufcnt: %d final: %d\n",
 668		__func__, rctx->total, rctx->state.bufcnt, final);
 
 
 669
 670	if (rctx->nents < 0)
 671		return -EINVAL;
 672
 673	stm32_hash_write_ctrl(hdev);
 674
 675	if (hdev->flags & HASH_FLAGS_HMAC && (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
 676		hdev->flags |= HASH_FLAGS_HMAC_KEY;
 677		err = stm32_hash_hmac_dma_send(hdev);
 678		if (err != -EINPROGRESS)
 679			return err;
 680	}
 681
 682	for_each_sg(rctx->sg, tsg, rctx->nents, i) {
 683		sg[0] = *tsg;
 684		len = sg->length;
 685
 686		if (sg_is_last(sg) || (bufcnt + sg[0].length) >= rctx->total) {
 687			if (!final) {
 688				/* Always manually put the last word of a non-final transfer. */
 689				len -= sizeof(u32);
 690				sg_pcopy_to_buffer(rctx->sg, rctx->nents, &last_word, 4, len);
 691				sg->length -= sizeof(u32);
 
 
 
 
 
 692			} else {
 693				/*
 694				 * In Multiple DMA mode, DMA must be aborted before the final
 695				 * transfer.
 696				 */
 697				sg->length = rctx->total - bufcnt;
 698				if (hdev->dma_mode > 0) {
 699					len = (ALIGN(sg->length, 16) - 16);
 700
 701					ncp = sg_pcopy_to_buffer(rctx->sg, rctx->nents,
 702								 rctx->state.buffer,
 703								 sg->length - len,
 704								 rctx->total - sg->length + len);
 705
 706					if (!len)
 707						break;
 708
 709					sg->length = len;
 710				} else {
 711					is_last = true;
 712					if (!(IS_ALIGNED(sg->length, sizeof(u32)))) {
 713						len = sg->length;
 714						sg->length = ALIGN(sg->length,
 715								   sizeof(u32));
 716					}
 717				}
 718			}
 719		}
 720
 721		rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1,
 722					  DMA_TO_DEVICE);
 723		if (rctx->dma_ct == 0) {
 724			dev_err(hdev->dev, "dma_map_sg error\n");
 725			return -ENOMEM;
 726		}
 727
 728		err = stm32_hash_xmit_dma(hdev, sg, len, !is_last);
 729
 730		/* The last word of a non final transfer is sent manually. */
 731		if (!final) {
 732			stm32_hash_write(hdev, HASH_DIN, last_word);
 733			len += sizeof(u32);
 734		}
 735
 736		rctx->total -= len;
 737
 738		bufcnt += sg[0].length;
 739		dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 740
 741		if (err == -ENOMEM || err == -ETIMEDOUT)
 742			return err;
 743		if (is_last)
 744			break;
 745	}
 746
 747	/*
 748	 * When the second last block transfer of 4 words is performed by the DMA,
 749	 * the software must set the DMA Abort bit (DMAA) to 1 before completing the
 750	 * last transfer of 4 words or less.
 751	 */
 752	if (final) {
 753		if (hdev->dma_mode > 0) {
 754			if (stm32_hash_wait_busy(hdev))
 755				return -ETIMEDOUT;
 756			reg = stm32_hash_read(hdev, HASH_CR);
 757			reg &= ~HASH_CR_DMAE;
 758			reg |= HASH_CR_DMAA;
 759			stm32_hash_write(hdev, HASH_CR, reg);
 760
 761			if (ncp) {
 762				memset(buffer + ncp, 0, 4 - DIV_ROUND_UP(ncp, sizeof(u32)));
 763				writesl(hdev->io_base + HASH_DIN, buffer,
 764					DIV_ROUND_UP(ncp, sizeof(u32)));
 765			}
 766
 767			stm32_hash_set_nblw(hdev, ncp);
 768			reg = stm32_hash_read(hdev, HASH_STR);
 769			reg |= HASH_STR_DCAL;
 770			stm32_hash_write(hdev, HASH_STR, reg);
 771			err = -EINPROGRESS;
 772		}
 773
 774		/*
 775		 * The hash processor needs the key to be loaded a second time in order
 776		 * to process the HMAC.
 777		 */
 778		if (hdev->flags & HASH_FLAGS_HMAC) {
 779			if (stm32_hash_wait_busy(hdev))
 780				return -ETIMEDOUT;
 781			err = stm32_hash_hmac_dma_send(hdev);
 782		}
 783
 784		return err;
 
 
 
 785	}
 786
 787	if (err != -EINPROGRESS)
 788		return err;
 
 
 
 789
 790	return 0;
 791}
 792
 793static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx)
 794{
 795	struct stm32_hash_dev *hdev = NULL, *tmp;
 796
 797	spin_lock_bh(&stm32_hash.lock);
 798	if (!ctx->hdev) {
 799		list_for_each_entry(tmp, &stm32_hash.dev_list, list) {
 800			hdev = tmp;
 801			break;
 802		}
 803		ctx->hdev = hdev;
 804	} else {
 805		hdev = ctx->hdev;
 806	}
 807
 808	spin_unlock_bh(&stm32_hash.lock);
 809
 810	return hdev;
 811}
 812
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 813static int stm32_hash_init(struct ahash_request *req)
 814{
 815	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 816	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 817	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 818	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
 819	struct stm32_hash_state *state = &rctx->state;
 820	bool sha3_mode = ctx->flags & HASH_FLAGS_SHA3_MODE;
 821
 822	rctx->hdev = hdev;
 823	state->flags = 0;
 824
 825	if (!(hdev->dma_lch &&  hdev->pdata->has_mdmat))
 826		state->flags |= HASH_FLAGS_CPU;
 827
 828	if (sha3_mode)
 829		state->flags |= HASH_FLAGS_SHA3_MODE;
 830
 831	rctx->digcnt = crypto_ahash_digestsize(tfm);
 832	switch (rctx->digcnt) {
 833	case MD5_DIGEST_SIZE:
 834		state->flags |= HASH_MD5 << HASH_FLAGS_ALGO_SHIFT;
 835		break;
 836	case SHA1_DIGEST_SIZE:
 837		if (hdev->pdata->ux500)
 838			state->flags |= HASH_SHA1_UX500 << HASH_FLAGS_ALGO_SHIFT;
 839		else
 840			state->flags |= HASH_SHA1 << HASH_FLAGS_ALGO_SHIFT;
 841		break;
 842	case SHA224_DIGEST_SIZE:
 843		if (sha3_mode)
 844			state->flags |= HASH_SHA3_224 << HASH_FLAGS_ALGO_SHIFT;
 845		else
 846			state->flags |= HASH_SHA224 << HASH_FLAGS_ALGO_SHIFT;
 847		break;
 848	case SHA256_DIGEST_SIZE:
 849		if (sha3_mode) {
 850			state->flags |= HASH_SHA3_256 << HASH_FLAGS_ALGO_SHIFT;
 851		} else {
 852			if (hdev->pdata->ux500)
 853				state->flags |= HASH_SHA256_UX500 << HASH_FLAGS_ALGO_SHIFT;
 854			else
 855				state->flags |= HASH_SHA256 << HASH_FLAGS_ALGO_SHIFT;
 856		}
 857		break;
 858	case SHA384_DIGEST_SIZE:
 859		if (sha3_mode)
 860			state->flags |= HASH_SHA3_384 << HASH_FLAGS_ALGO_SHIFT;
 861		else
 862			state->flags |= HASH_SHA384 << HASH_FLAGS_ALGO_SHIFT;
 863		break;
 864	case SHA512_DIGEST_SIZE:
 865		if (sha3_mode)
 866			state->flags |= HASH_SHA3_512 << HASH_FLAGS_ALGO_SHIFT;
 867		else
 868			state->flags |= HASH_SHA512 << HASH_FLAGS_ALGO_SHIFT;
 869		break;
 870	default:
 871		return -EINVAL;
 872	}
 873
 874	rctx->state.bufcnt = 0;
 875	rctx->state.blocklen = crypto_ahash_blocksize(tfm) + sizeof(u32);
 876	if (rctx->state.blocklen > HASH_BUFLEN) {
 877		dev_err(hdev->dev, "Error, block too large");
 878		return -EINVAL;
 879	}
 880	rctx->nents = 0;
 881	rctx->total = 0;
 882	rctx->offset = 0;
 883	rctx->data_type = HASH_DATA_8_BITS;
 884
 
 
 885	if (ctx->flags & HASH_FLAGS_HMAC)
 886		state->flags |= HASH_FLAGS_HMAC;
 887
 888	dev_dbg(hdev->dev, "%s Flags %x\n", __func__, state->flags);
 889
 890	return 0;
 891}
 892
 893static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
 894{
 895	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 896	struct stm32_hash_state *state = &rctx->state;
 897
 898	dev_dbg(hdev->dev, "update_req: total: %u, digcnt: %zd, final: 0",
 899		rctx->total, rctx->digcnt);
 900
 901	if (!(state->flags & HASH_FLAGS_CPU))
 902		return stm32_hash_dma_send(hdev);
 903
 904	return stm32_hash_update_cpu(hdev);
 905}
 906
 907static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
 908{
 909	struct ahash_request *req = hdev->req;
 910	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 911	struct stm32_hash_state *state = &rctx->state;
 912	int buflen = state->bufcnt;
 913
 914	if (!(state->flags & HASH_FLAGS_CPU)) {
 915		hdev->flags |= HASH_FLAGS_FINAL;
 916		return stm32_hash_dma_send(hdev);
 917	}
 918
 919	if (state->flags & HASH_FLAGS_FINUP)
 920		return stm32_hash_update_req(hdev);
 921
 922	state->bufcnt = 0;
 923
 924	return stm32_hash_xmit_cpu(hdev, state->buffer, buflen, 1);
 925}
 926
 927static void stm32_hash_emptymsg_fallback(struct ahash_request *req)
 928{
 929	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 930	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 931	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 932	struct stm32_hash_dev *hdev = rctx->hdev;
 933	int ret;
 934
 935	dev_dbg(hdev->dev, "use fallback message size 0 key size %d\n",
 936		ctx->keylen);
 937
 938	if (!ctx->xtfm) {
 939		dev_err(hdev->dev, "no fallback engine\n");
 940		return;
 941	}
 942
 943	if (ctx->keylen) {
 944		ret = crypto_shash_setkey(ctx->xtfm, ctx->key, ctx->keylen);
 945		if (ret) {
 946			dev_err(hdev->dev, "failed to set key ret=%d\n", ret);
 947			return;
 948		}
 949	}
 950
 951	ret = crypto_shash_tfm_digest(ctx->xtfm, NULL, 0, rctx->digest);
 952	if (ret)
 953		dev_err(hdev->dev, "shash digest error\n");
 954}
 955
 956static void stm32_hash_copy_hash(struct ahash_request *req)
 957{
 958	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 959	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 960	struct stm32_hash_state *state = &rctx->state;
 961	struct stm32_hash_dev *hdev = rctx->hdev;
 962	__be32 *hash = (void *)rctx->digest;
 963	unsigned int i, hashsize;
 964
 965	if (hdev->pdata->broken_emptymsg && (state->flags & HASH_FLAGS_EMPTY))
 966		return stm32_hash_emptymsg_fallback(req);
 967
 968	hashsize = crypto_ahash_digestsize(tfm);
 969
 970	for (i = 0; i < hashsize / sizeof(u32); i++) {
 971		if (hdev->pdata->ux500)
 972			hash[i] = cpu_to_be32(stm32_hash_read(hdev,
 973					      HASH_UX500_HREG(i)));
 974		else
 975			hash[i] = cpu_to_be32(stm32_hash_read(hdev,
 976					      HASH_HREG(i)));
 
 
 
 977	}
 
 
 
 
 978}
 979
 980static int stm32_hash_finish(struct ahash_request *req)
 981{
 982	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 983	u32 reg;
 984
 985	reg = stm32_hash_read(rctx->hdev, HASH_SR);
 986	reg &= ~HASH_SR_OUTPUT_READY;
 987	stm32_hash_write(rctx->hdev, HASH_SR, reg);
 988
 989	if (!req->result)
 990		return -EINVAL;
 991
 992	memcpy(req->result, rctx->digest, rctx->digcnt);
 993
 994	return 0;
 995}
 996
 997static void stm32_hash_finish_req(struct ahash_request *req, int err)
 998{
 999	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1000	struct stm32_hash_state *state = &rctx->state;
1001	struct stm32_hash_dev *hdev = rctx->hdev;
1002
1003	if (hdev->flags & HASH_FLAGS_DMA_ACTIVE)
1004		state->flags |= HASH_FLAGS_DMA_ACTIVE;
1005	else
1006		state->flags &= ~HASH_FLAGS_DMA_ACTIVE;
1007
1008	if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
1009		stm32_hash_copy_hash(req);
1010		err = stm32_hash_finish(req);
 
 
 
 
 
 
 
1011	}
1012
1013	/* Finalized request mist be unprepared here */
1014	stm32_hash_unprepare_request(req);
1015
1016	crypto_finalize_hash_request(hdev->engine, req, err);
1017}
1018
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1019static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
1020				   struct ahash_request *req)
1021{
1022	return crypto_transfer_hash_request_to_engine(hdev->engine, req);
1023}
1024
1025static int stm32_hash_one_request(struct crypto_engine *engine, void *areq)
1026{
1027	struct ahash_request *req = container_of(areq, struct ahash_request,
1028						 base);
1029	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1030	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1031	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1032	struct stm32_hash_state *state = &rctx->state;
1033	int swap_reg;
1034	int err = 0;
1035
1036	if (!hdev)
1037		return -ENODEV;
1038
1039	dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
1040		rctx->op, req->nbytes);
1041
1042	pm_runtime_get_sync(hdev->dev);
1043
1044	err = stm32_hash_prepare_request(req);
1045	if (err)
1046		return err;
1047
1048	hdev->req = req;
1049	hdev->flags = 0;
1050	swap_reg = hash_swap_reg(rctx);
1051
1052	if (state->flags & HASH_FLAGS_INIT) {
1053		u32 *preg = rctx->state.hw_context;
1054		u32 reg;
1055		int i;
1056
1057		if (!hdev->pdata->ux500)
1058			stm32_hash_write(hdev, HASH_IMR, *preg++);
1059		stm32_hash_write(hdev, HASH_STR, *preg++);
1060		stm32_hash_write(hdev, HASH_CR, *preg);
1061		reg = *preg++ | HASH_CR_INIT;
1062		stm32_hash_write(hdev, HASH_CR, reg);
1063
1064		for (i = 0; i < swap_reg; i++)
1065			stm32_hash_write(hdev, HASH_CSR(i), *preg++);
1066
1067		hdev->flags |= HASH_FLAGS_INIT;
 
1068
1069		if (state->flags & HASH_FLAGS_HMAC)
1070			hdev->flags |= HASH_FLAGS_HMAC |
1071				       HASH_FLAGS_HMAC_KEY;
 
 
 
 
 
1072
1073		if (state->flags & HASH_FLAGS_CPU)
1074			hdev->flags |= HASH_FLAGS_CPU;
1075
1076		if (state->flags & HASH_FLAGS_DMA_ACTIVE)
1077			hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
1078	}
1079
1080	if (rctx->op == HASH_OP_UPDATE)
1081		err = stm32_hash_update_req(hdev);
1082	else if (rctx->op == HASH_OP_FINAL)
1083		err = stm32_hash_final_req(hdev);
1084
1085	/* If we have an IRQ, wait for that, else poll for completion */
1086	if (err == -EINPROGRESS && hdev->polled) {
1087		if (stm32_hash_wait_busy(hdev))
1088			err = -ETIMEDOUT;
1089		else {
1090			hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1091			err = 0;
1092		}
1093	}
1094
1095	if (err != -EINPROGRESS)
1096	/* done task will not finish it, so do it here */
1097		stm32_hash_finish_req(req, err);
1098
1099	return 0;
1100}
1101
1102static int stm32_hash_copy_sgs(struct stm32_hash_request_ctx *rctx,
1103			       struct scatterlist *sg, int bs,
1104			       unsigned int new_len)
1105{
1106	struct stm32_hash_state *state = &rctx->state;
1107	int pages;
1108	void *buf;
1109
1110	pages = get_order(new_len);
1111
1112	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
1113	if (!buf) {
1114		pr_err("Couldn't allocate pages for unaligned cases.\n");
1115		return -ENOMEM;
1116	}
1117
1118	if (state->bufcnt)
1119		memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt);
1120
1121	scatterwalk_map_and_copy(buf + state->bufcnt, sg, rctx->offset,
1122				 min(new_len, rctx->total) - state->bufcnt, 0);
1123	sg_init_table(rctx->sgl, 1);
1124	sg_set_buf(rctx->sgl, buf, new_len);
1125	rctx->sg = rctx->sgl;
1126	state->flags |= HASH_FLAGS_SGS_COPIED;
1127	rctx->nents = 1;
1128	rctx->offset += new_len - state->bufcnt;
1129	state->bufcnt = 0;
1130	rctx->total = new_len;
1131
1132	return 0;
1133}
1134
1135static int stm32_hash_align_sgs(struct scatterlist *sg,
1136				int nbytes, int bs, bool init, bool final,
1137				struct stm32_hash_request_ctx *rctx)
1138{
1139	struct stm32_hash_state *state = &rctx->state;
1140	struct stm32_hash_dev *hdev = rctx->hdev;
1141	struct scatterlist *sg_tmp = sg;
1142	int offset = rctx->offset;
1143	int new_len;
1144	int n = 0;
1145	int bufcnt = state->bufcnt;
1146	bool secure_ctx = hdev->pdata->context_secured;
1147	bool aligned = true;
1148
1149	if (!sg || !sg->length || !nbytes) {
1150		if (bufcnt) {
1151			bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
1152			sg_init_table(rctx->sgl, 1);
1153			sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, bufcnt);
1154			rctx->sg = rctx->sgl;
1155			rctx->nents = 1;
1156		}
1157
1158		return 0;
1159	}
1160
1161	new_len = nbytes;
1162
1163	if (offset)
1164		aligned = false;
1165
1166	if (final) {
1167		new_len = DIV_ROUND_UP(new_len, bs) * bs;
1168	} else {
1169		new_len = (new_len - 1) / bs * bs; // return n block - 1 block
1170
1171		/*
1172		 * Context save in some version of HASH IP can only be done when the
1173		 * FIFO is ready to get a new block. This implies to send n block plus a
1174		 * 32 bit word in the first DMA send.
1175		 */
1176		if (init && secure_ctx) {
1177			new_len += sizeof(u32);
1178			if (unlikely(new_len > nbytes))
1179				new_len -= bs;
1180		}
1181	}
1182
1183	if (!new_len)
1184		return 0;
1185
1186	if (nbytes != new_len)
1187		aligned = false;
1188
1189	while (nbytes > 0 && sg_tmp) {
1190		n++;
1191
1192		if (bufcnt) {
1193			if (!IS_ALIGNED(bufcnt, bs)) {
1194				aligned = false;
1195				break;
1196			}
1197			nbytes -= bufcnt;
1198			bufcnt = 0;
1199			if (!nbytes)
1200				aligned = false;
1201
1202			continue;
1203		}
1204
1205		if (offset < sg_tmp->length) {
1206			if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
1207				aligned = false;
1208				break;
1209			}
1210
1211			if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
1212				aligned = false;
1213				break;
1214			}
1215		}
1216
1217		if (offset) {
1218			offset -= sg_tmp->length;
1219			if (offset < 0) {
1220				nbytes += offset;
1221				offset = 0;
1222			}
1223		} else {
1224			nbytes -= sg_tmp->length;
1225		}
1226
1227		sg_tmp = sg_next(sg_tmp);
1228
1229		if (nbytes < 0) {
1230			aligned = false;
1231			break;
1232		}
1233	}
1234
1235	if (!aligned)
1236		return stm32_hash_copy_sgs(rctx, sg, bs, new_len);
1237
1238	rctx->total = new_len;
1239	rctx->offset += new_len;
1240	rctx->nents = n;
1241	if (state->bufcnt) {
1242		sg_init_table(rctx->sgl, 2);
1243		sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, state->bufcnt);
1244		sg_chain(rctx->sgl, 2, sg);
1245		rctx->sg = rctx->sgl;
1246	} else {
1247		rctx->sg = sg;
1248	}
1249
1250	return 0;
1251}
1252
1253static int stm32_hash_prepare_request(struct ahash_request *req)
1254{
1255	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1256	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1257	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1258	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1259	struct stm32_hash_state *state = &rctx->state;
1260	unsigned int nbytes;
1261	int ret, hash_later, bs;
1262	bool update = rctx->op & HASH_OP_UPDATE;
1263	bool init = !(state->flags & HASH_FLAGS_INIT);
1264	bool finup = state->flags & HASH_FLAGS_FINUP;
1265	bool final = state->flags & HASH_FLAGS_FINAL;
1266
1267	if (!hdev->dma_lch || state->flags & HASH_FLAGS_CPU)
1268		return 0;
1269
1270	bs = crypto_ahash_blocksize(tfm);
1271
1272	nbytes = state->bufcnt;
1273
1274	/*
1275	 * In case of update request nbytes must correspond to the content of the
1276	 * buffer + the offset minus the content of the request already in the
1277	 * buffer.
1278	 */
1279	if (update || finup)
1280		nbytes += req->nbytes - rctx->offset;
1281
1282	dev_dbg(hdev->dev,
1283		"%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%d\n",
1284		__func__, nbytes, bs, rctx->total, rctx->offset, state->bufcnt);
1285
1286	if (!nbytes)
1287		return 0;
1288
1289	rctx->total = nbytes;
1290
1291	if (update && req->nbytes && (!IS_ALIGNED(state->bufcnt, bs))) {
1292		int len = bs - state->bufcnt % bs;
1293
1294		if (len > req->nbytes)
1295			len = req->nbytes;
1296		scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src,
1297					 0, len, 0);
1298		state->bufcnt += len;
1299		rctx->offset = len;
1300	}
1301
1302	/* copy buffer in a temporary one that is used for sg alignment */
1303	if (state->bufcnt)
1304		memcpy(hdev->xmit_buf, state->buffer, state->bufcnt);
1305
1306	ret = stm32_hash_align_sgs(req->src, nbytes, bs, init, final, rctx);
1307	if (ret)
1308		return ret;
1309
1310	hash_later = nbytes - rctx->total;
1311	if (hash_later < 0)
1312		hash_later = 0;
1313
1314	if (hash_later && hash_later <= state->blocklen) {
1315		scatterwalk_map_and_copy(state->buffer,
1316					 req->src,
1317					 req->nbytes - hash_later,
1318					 hash_later, 0);
1319
1320		state->bufcnt = hash_later;
1321	} else {
1322		state->bufcnt = 0;
1323	}
1324
1325	if (hash_later > state->blocklen) {
1326		/* FIXME: add support of this case */
1327		pr_err("Buffer contains more than one block.\n");
1328		return -ENOMEM;
1329	}
1330
1331	rctx->total = min(nbytes, rctx->total);
1332
1333	return 0;
1334}
1335
1336static void stm32_hash_unprepare_request(struct ahash_request *req)
1337{
1338	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1339	struct stm32_hash_state *state = &rctx->state;
1340	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1341	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1342	u32 *preg = state->hw_context;
1343	int swap_reg, i;
1344
1345	if (hdev->dma_lch)
1346		dmaengine_terminate_sync(hdev->dma_lch);
1347
1348	if (state->flags & HASH_FLAGS_SGS_COPIED)
1349		free_pages((unsigned long)sg_virt(rctx->sg), get_order(rctx->sg->length));
1350
1351	rctx->sg = NULL;
1352	rctx->offset = 0;
1353
1354	state->flags &= ~(HASH_FLAGS_SGS_COPIED);
1355
1356	if (!(hdev->flags & HASH_FLAGS_INIT))
1357		goto pm_runtime;
1358
1359	state->flags |= HASH_FLAGS_INIT;
1360
1361	if (stm32_hash_wait_busy(hdev)) {
1362		dev_warn(hdev->dev, "Wait busy failed.");
1363		return;
1364	}
1365
1366	swap_reg = hash_swap_reg(rctx);
1367
1368	if (!hdev->pdata->ux500)
1369		*preg++ = stm32_hash_read(hdev, HASH_IMR);
1370	*preg++ = stm32_hash_read(hdev, HASH_STR);
1371	*preg++ = stm32_hash_read(hdev, HASH_CR);
1372	for (i = 0; i < swap_reg; i++)
1373		*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
1374
1375pm_runtime:
1376	pm_runtime_mark_last_busy(hdev->dev);
1377	pm_runtime_put_autosuspend(hdev->dev);
1378}
1379
1380static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
1381{
1382	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1383	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1384	struct stm32_hash_dev *hdev = ctx->hdev;
1385
1386	rctx->op = op;
1387
1388	return stm32_hash_handle_queue(hdev, req);
1389}
1390
1391static int stm32_hash_update(struct ahash_request *req)
1392{
1393	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1394	struct stm32_hash_state *state = &rctx->state;
1395
1396	if (!req->nbytes)
1397		return 0;
1398
 
 
 
1399
1400	if (state->flags & HASH_FLAGS_CPU) {
1401		rctx->total = req->nbytes;
1402		rctx->sg = req->src;
1403		rctx->offset = 0;
1404
1405		if ((state->bufcnt + rctx->total < state->blocklen)) {
1406			stm32_hash_append_sg(rctx);
1407			return 0;
1408		}
1409	} else { /* DMA mode */
1410		if (state->bufcnt + req->nbytes <= state->blocklen) {
1411			scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src,
1412						 0, req->nbytes, 0);
1413			state->bufcnt += req->nbytes;
1414			return 0;
1415		}
1416	}
1417
1418	return stm32_hash_enqueue(req, HASH_OP_UPDATE);
1419}
1420
1421static int stm32_hash_final(struct ahash_request *req)
1422{
1423	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1424	struct stm32_hash_state *state = &rctx->state;
1425
1426	state->flags |= HASH_FLAGS_FINAL;
1427
1428	return stm32_hash_enqueue(req, HASH_OP_FINAL);
1429}
1430
1431static int stm32_hash_finup(struct ahash_request *req)
1432{
1433	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1434	struct stm32_hash_state *state = &rctx->state;
 
 
1435
1436	if (!req->nbytes)
1437		goto out;
1438
1439	state->flags |= HASH_FLAGS_FINUP;
 
1440
1441	if ((state->flags & HASH_FLAGS_CPU)) {
1442		rctx->total = req->nbytes;
1443		rctx->sg = req->src;
1444		rctx->offset = 0;
1445	}
 
 
 
 
 
1446
1447out:
1448	return stm32_hash_final(req);
1449}
1450
1451static int stm32_hash_digest(struct ahash_request *req)
1452{
1453	return stm32_hash_init(req) ?: stm32_hash_finup(req);
1454}
1455
1456static int stm32_hash_export(struct ahash_request *req, void *out)
1457{
1458	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 
 
 
 
 
 
 
 
 
 
1459
1460	memcpy(out, &rctx->state, sizeof(rctx->state));
 
 
 
 
 
 
 
 
1461
1462	return 0;
1463}
1464
1465static int stm32_hash_import(struct ahash_request *req, const void *in)
1466{
1467	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1468
1469	stm32_hash_init(req);
1470	memcpy(&rctx->state, in, sizeof(rctx->state));
1471
1472	return 0;
1473}
1474
1475static int stm32_hash_setkey(struct crypto_ahash *tfm,
1476			     const u8 *key, unsigned int keylen)
1477{
1478	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1479
1480	if (keylen <= HASH_MAX_KEY_SIZE) {
1481		memcpy(ctx->key, key, keylen);
1482		ctx->keylen = keylen;
1483	} else {
1484		return -ENOMEM;
1485	}
1486
1487	return 0;
1488}
1489
1490static int stm32_hash_init_fallback(struct crypto_tfm *tfm)
1491{
1492	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1493	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1494	const char *name = crypto_tfm_alg_name(tfm);
1495	struct crypto_shash *xtfm;
1496
1497	/* The fallback is only needed on Ux500 */
1498	if (!hdev->pdata->ux500)
1499		return 0;
1500
1501	xtfm = crypto_alloc_shash(name, 0, CRYPTO_ALG_NEED_FALLBACK);
1502	if (IS_ERR(xtfm)) {
1503		dev_err(hdev->dev, "failed to allocate %s fallback\n",
1504			name);
1505		return PTR_ERR(xtfm);
1506	}
1507	dev_info(hdev->dev, "allocated %s fallback\n", name);
1508	ctx->xtfm = xtfm;
1509
1510	return 0;
1511}
1512
1513static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm, u32 algs_flags)
1514{
1515	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1516
1517	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1518				 sizeof(struct stm32_hash_request_ctx));
1519
1520	ctx->keylen = 0;
1521
1522	if (algs_flags)
1523		ctx->flags |= algs_flags;
1524
1525	return stm32_hash_init_fallback(tfm);
 
 
 
1526}
1527
1528static int stm32_hash_cra_init(struct crypto_tfm *tfm)
1529{
1530	return stm32_hash_cra_init_algs(tfm, 0);
1531}
1532
1533static int stm32_hash_cra_hmac_init(struct crypto_tfm *tfm)
1534{
1535	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_HMAC);
1536}
1537
1538static int stm32_hash_cra_sha3_init(struct crypto_tfm *tfm)
1539{
1540	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE);
1541}
1542
1543static int stm32_hash_cra_sha3_hmac_init(struct crypto_tfm *tfm)
1544{
1545	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE |
1546					HASH_FLAGS_HMAC);
1547}
1548
1549static void stm32_hash_cra_exit(struct crypto_tfm *tfm)
1550{
1551	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1552
1553	if (ctx->xtfm)
1554		crypto_free_shash(ctx->xtfm);
1555}
1556
1557static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id)
1558{
1559	struct stm32_hash_dev *hdev = dev_id;
1560
1561	if (HASH_FLAGS_OUTPUT_READY & hdev->flags) {
1562		hdev->flags &= ~HASH_FLAGS_OUTPUT_READY;
1563		goto finish;
 
 
 
 
 
 
 
1564	}
1565
1566	return IRQ_HANDLED;
1567
1568finish:
1569	/* Finish current request */
1570	stm32_hash_finish_req(hdev->req, 0);
1571
1572	return IRQ_HANDLED;
1573}
1574
1575static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id)
1576{
1577	struct stm32_hash_dev *hdev = dev_id;
1578	u32 reg;
1579
1580	reg = stm32_hash_read(hdev, HASH_SR);
1581	if (reg & HASH_SR_OUTPUT_READY) {
 
 
1582		hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1583		/* Disable IT*/
1584		stm32_hash_write(hdev, HASH_IMR, 0);
1585		return IRQ_WAKE_THREAD;
1586	}
1587
1588	return IRQ_NONE;
1589}
1590
1591static struct ahash_engine_alg algs_md5[] = {
1592	{
1593		.base.init = stm32_hash_init,
1594		.base.update = stm32_hash_update,
1595		.base.final = stm32_hash_final,
1596		.base.finup = stm32_hash_finup,
1597		.base.digest = stm32_hash_digest,
1598		.base.export = stm32_hash_export,
1599		.base.import = stm32_hash_import,
1600		.base.halg = {
1601			.digestsize = MD5_DIGEST_SIZE,
1602			.statesize = sizeof(struct stm32_hash_state),
1603			.base = {
1604				.cra_name = "md5",
1605				.cra_driver_name = "stm32-md5",
1606				.cra_priority = 200,
1607				.cra_flags = CRYPTO_ALG_ASYNC |
 
1608					CRYPTO_ALG_KERN_DRIVER_ONLY,
1609				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1610				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
 
1611				.cra_init = stm32_hash_cra_init,
1612				.cra_exit = stm32_hash_cra_exit,
1613				.cra_module = THIS_MODULE,
1614			}
1615		},
1616		.op = {
1617			.do_one_request = stm32_hash_one_request,
1618		},
1619	},
1620	{
1621		.base.init = stm32_hash_init,
1622		.base.update = stm32_hash_update,
1623		.base.final = stm32_hash_final,
1624		.base.finup = stm32_hash_finup,
1625		.base.digest = stm32_hash_digest,
1626		.base.export = stm32_hash_export,
1627		.base.import = stm32_hash_import,
1628		.base.setkey = stm32_hash_setkey,
1629		.base.halg = {
1630			.digestsize = MD5_DIGEST_SIZE,
1631			.statesize = sizeof(struct stm32_hash_state),
1632			.base = {
1633				.cra_name = "hmac(md5)",
1634				.cra_driver_name = "stm32-hmac-md5",
1635				.cra_priority = 200,
1636				.cra_flags = CRYPTO_ALG_ASYNC |
 
1637					CRYPTO_ALG_KERN_DRIVER_ONLY,
1638				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1639				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1640				.cra_init = stm32_hash_cra_hmac_init,
1641				.cra_exit = stm32_hash_cra_exit,
1642				.cra_module = THIS_MODULE,
1643			}
1644		},
1645		.op = {
1646			.do_one_request = stm32_hash_one_request,
1647		},
1648	}
1649};
1650
1651static struct ahash_engine_alg algs_sha1[] = {
1652	{
1653		.base.init = stm32_hash_init,
1654		.base.update = stm32_hash_update,
1655		.base.final = stm32_hash_final,
1656		.base.finup = stm32_hash_finup,
1657		.base.digest = stm32_hash_digest,
1658		.base.export = stm32_hash_export,
1659		.base.import = stm32_hash_import,
1660		.base.halg = {
1661			.digestsize = SHA1_DIGEST_SIZE,
1662			.statesize = sizeof(struct stm32_hash_state),
1663			.base = {
1664				.cra_name = "sha1",
1665				.cra_driver_name = "stm32-sha1",
1666				.cra_priority = 200,
1667				.cra_flags = CRYPTO_ALG_ASYNC |
 
1668					CRYPTO_ALG_KERN_DRIVER_ONLY,
1669				.cra_blocksize = SHA1_BLOCK_SIZE,
1670				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
 
1671				.cra_init = stm32_hash_cra_init,
1672				.cra_exit = stm32_hash_cra_exit,
1673				.cra_module = THIS_MODULE,
1674			}
1675		},
1676		.op = {
1677			.do_one_request = stm32_hash_one_request,
1678		},
1679	},
1680	{
1681		.base.init = stm32_hash_init,
1682		.base.update = stm32_hash_update,
1683		.base.final = stm32_hash_final,
1684		.base.finup = stm32_hash_finup,
1685		.base.digest = stm32_hash_digest,
1686		.base.export = stm32_hash_export,
1687		.base.import = stm32_hash_import,
1688		.base.setkey = stm32_hash_setkey,
1689		.base.halg = {
1690			.digestsize = SHA1_DIGEST_SIZE,
1691			.statesize = sizeof(struct stm32_hash_state),
1692			.base = {
1693				.cra_name = "hmac(sha1)",
1694				.cra_driver_name = "stm32-hmac-sha1",
1695				.cra_priority = 200,
1696				.cra_flags = CRYPTO_ALG_ASYNC |
 
1697					CRYPTO_ALG_KERN_DRIVER_ONLY,
1698				.cra_blocksize = SHA1_BLOCK_SIZE,
1699				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1700				.cra_init = stm32_hash_cra_hmac_init,
1701				.cra_exit = stm32_hash_cra_exit,
1702				.cra_module = THIS_MODULE,
1703			}
1704		},
1705		.op = {
1706			.do_one_request = stm32_hash_one_request,
1707		},
1708	},
1709};
1710
1711static struct ahash_engine_alg algs_sha224[] = {
1712	{
1713		.base.init = stm32_hash_init,
1714		.base.update = stm32_hash_update,
1715		.base.final = stm32_hash_final,
1716		.base.finup = stm32_hash_finup,
1717		.base.digest = stm32_hash_digest,
1718		.base.export = stm32_hash_export,
1719		.base.import = stm32_hash_import,
1720		.base.halg = {
1721			.digestsize = SHA224_DIGEST_SIZE,
1722			.statesize = sizeof(struct stm32_hash_state),
1723			.base = {
1724				.cra_name = "sha224",
1725				.cra_driver_name = "stm32-sha224",
1726				.cra_priority = 200,
1727				.cra_flags = CRYPTO_ALG_ASYNC |
 
1728					CRYPTO_ALG_KERN_DRIVER_ONLY,
1729				.cra_blocksize = SHA224_BLOCK_SIZE,
1730				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
 
1731				.cra_init = stm32_hash_cra_init,
1732				.cra_exit = stm32_hash_cra_exit,
1733				.cra_module = THIS_MODULE,
1734			}
1735		},
1736		.op = {
1737			.do_one_request = stm32_hash_one_request,
1738		},
1739	},
1740	{
1741		.base.init = stm32_hash_init,
1742		.base.update = stm32_hash_update,
1743		.base.final = stm32_hash_final,
1744		.base.finup = stm32_hash_finup,
1745		.base.digest = stm32_hash_digest,
1746		.base.setkey = stm32_hash_setkey,
1747		.base.export = stm32_hash_export,
1748		.base.import = stm32_hash_import,
1749		.base.halg = {
1750			.digestsize = SHA224_DIGEST_SIZE,
1751			.statesize = sizeof(struct stm32_hash_state),
1752			.base = {
1753				.cra_name = "hmac(sha224)",
1754				.cra_driver_name = "stm32-hmac-sha224",
1755				.cra_priority = 200,
1756				.cra_flags = CRYPTO_ALG_ASYNC |
 
1757					CRYPTO_ALG_KERN_DRIVER_ONLY,
1758				.cra_blocksize = SHA224_BLOCK_SIZE,
1759				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1760				.cra_init = stm32_hash_cra_hmac_init,
1761				.cra_exit = stm32_hash_cra_exit,
1762				.cra_module = THIS_MODULE,
1763			}
1764		},
1765		.op = {
1766			.do_one_request = stm32_hash_one_request,
1767		},
1768	},
1769};
1770
1771static struct ahash_engine_alg algs_sha256[] = {
1772	{
1773		.base.init = stm32_hash_init,
1774		.base.update = stm32_hash_update,
1775		.base.final = stm32_hash_final,
1776		.base.finup = stm32_hash_finup,
1777		.base.digest = stm32_hash_digest,
1778		.base.export = stm32_hash_export,
1779		.base.import = stm32_hash_import,
1780		.base.halg = {
1781			.digestsize = SHA256_DIGEST_SIZE,
1782			.statesize = sizeof(struct stm32_hash_state),
1783			.base = {
1784				.cra_name = "sha256",
1785				.cra_driver_name = "stm32-sha256",
1786				.cra_priority = 200,
1787				.cra_flags = CRYPTO_ALG_ASYNC |
 
1788					CRYPTO_ALG_KERN_DRIVER_ONLY,
1789				.cra_blocksize = SHA256_BLOCK_SIZE,
1790				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
 
1791				.cra_init = stm32_hash_cra_init,
1792				.cra_exit = stm32_hash_cra_exit,
1793				.cra_module = THIS_MODULE,
1794			}
1795		},
1796		.op = {
1797			.do_one_request = stm32_hash_one_request,
1798		},
1799	},
1800	{
1801		.base.init = stm32_hash_init,
1802		.base.update = stm32_hash_update,
1803		.base.final = stm32_hash_final,
1804		.base.finup = stm32_hash_finup,
1805		.base.digest = stm32_hash_digest,
1806		.base.export = stm32_hash_export,
1807		.base.import = stm32_hash_import,
1808		.base.setkey = stm32_hash_setkey,
1809		.base.halg = {
1810			.digestsize = SHA256_DIGEST_SIZE,
1811			.statesize = sizeof(struct stm32_hash_state),
1812			.base = {
1813				.cra_name = "hmac(sha256)",
1814				.cra_driver_name = "stm32-hmac-sha256",
1815				.cra_priority = 200,
1816				.cra_flags = CRYPTO_ALG_ASYNC |
 
1817					CRYPTO_ALG_KERN_DRIVER_ONLY,
1818				.cra_blocksize = SHA256_BLOCK_SIZE,
1819				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1820				.cra_init = stm32_hash_cra_hmac_init,
1821				.cra_exit = stm32_hash_cra_exit,
1822				.cra_module = THIS_MODULE,
1823			}
1824		},
1825		.op = {
1826			.do_one_request = stm32_hash_one_request,
1827		},
1828	},
1829};
1830
1831static struct ahash_engine_alg algs_sha384_sha512[] = {
1832	{
1833		.base.init = stm32_hash_init,
1834		.base.update = stm32_hash_update,
1835		.base.final = stm32_hash_final,
1836		.base.finup = stm32_hash_finup,
1837		.base.digest = stm32_hash_digest,
1838		.base.export = stm32_hash_export,
1839		.base.import = stm32_hash_import,
1840		.base.halg = {
1841			.digestsize = SHA384_DIGEST_SIZE,
1842			.statesize = sizeof(struct stm32_hash_state),
1843			.base = {
1844				.cra_name = "sha384",
1845				.cra_driver_name = "stm32-sha384",
1846				.cra_priority = 200,
1847				.cra_flags = CRYPTO_ALG_ASYNC |
1848					CRYPTO_ALG_KERN_DRIVER_ONLY,
1849				.cra_blocksize = SHA384_BLOCK_SIZE,
1850				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1851				.cra_init = stm32_hash_cra_init,
1852				.cra_exit = stm32_hash_cra_exit,
1853				.cra_module = THIS_MODULE,
1854			}
1855		},
1856		.op = {
1857			.do_one_request = stm32_hash_one_request,
1858		},
1859	},
1860	{
1861		.base.init = stm32_hash_init,
1862		.base.update = stm32_hash_update,
1863		.base.final = stm32_hash_final,
1864		.base.finup = stm32_hash_finup,
1865		.base.digest = stm32_hash_digest,
1866		.base.setkey = stm32_hash_setkey,
1867		.base.export = stm32_hash_export,
1868		.base.import = stm32_hash_import,
1869		.base.halg = {
1870			.digestsize = SHA384_DIGEST_SIZE,
1871			.statesize = sizeof(struct stm32_hash_state),
1872			.base = {
1873				.cra_name = "hmac(sha384)",
1874				.cra_driver_name = "stm32-hmac-sha384",
1875				.cra_priority = 200,
1876				.cra_flags = CRYPTO_ALG_ASYNC |
1877					CRYPTO_ALG_KERN_DRIVER_ONLY,
1878				.cra_blocksize = SHA384_BLOCK_SIZE,
1879				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1880				.cra_init = stm32_hash_cra_hmac_init,
1881				.cra_exit = stm32_hash_cra_exit,
1882				.cra_module = THIS_MODULE,
1883			}
1884		},
1885		.op = {
1886			.do_one_request = stm32_hash_one_request,
1887		},
1888	},
1889	{
1890		.base.init = stm32_hash_init,
1891		.base.update = stm32_hash_update,
1892		.base.final = stm32_hash_final,
1893		.base.finup = stm32_hash_finup,
1894		.base.digest = stm32_hash_digest,
1895		.base.export = stm32_hash_export,
1896		.base.import = stm32_hash_import,
1897		.base.halg = {
1898			.digestsize = SHA512_DIGEST_SIZE,
1899			.statesize = sizeof(struct stm32_hash_state),
1900			.base = {
1901				.cra_name = "sha512",
1902				.cra_driver_name = "stm32-sha512",
1903				.cra_priority = 200,
1904				.cra_flags = CRYPTO_ALG_ASYNC |
1905					CRYPTO_ALG_KERN_DRIVER_ONLY,
1906				.cra_blocksize = SHA512_BLOCK_SIZE,
1907				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1908				.cra_init = stm32_hash_cra_init,
1909				.cra_exit = stm32_hash_cra_exit,
1910				.cra_module = THIS_MODULE,
1911			}
1912		},
1913		.op = {
1914			.do_one_request = stm32_hash_one_request,
1915		},
1916	},
1917	{
1918		.base.init = stm32_hash_init,
1919		.base.update = stm32_hash_update,
1920		.base.final = stm32_hash_final,
1921		.base.finup = stm32_hash_finup,
1922		.base.digest = stm32_hash_digest,
1923		.base.export = stm32_hash_export,
1924		.base.import = stm32_hash_import,
1925		.base.setkey = stm32_hash_setkey,
1926		.base.halg = {
1927			.digestsize = SHA512_DIGEST_SIZE,
1928			.statesize = sizeof(struct stm32_hash_state),
1929			.base = {
1930				.cra_name = "hmac(sha512)",
1931				.cra_driver_name = "stm32-hmac-sha512",
1932				.cra_priority = 200,
1933				.cra_flags = CRYPTO_ALG_ASYNC |
1934					CRYPTO_ALG_KERN_DRIVER_ONLY,
1935				.cra_blocksize = SHA512_BLOCK_SIZE,
1936				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1937				.cra_init = stm32_hash_cra_hmac_init,
1938				.cra_exit = stm32_hash_cra_exit,
1939				.cra_module = THIS_MODULE,
1940			}
1941		},
1942		.op = {
1943			.do_one_request = stm32_hash_one_request,
1944		},
1945	},
1946};
1947
1948static struct ahash_engine_alg algs_sha3[] = {
1949	{
1950		.base.init = stm32_hash_init,
1951		.base.update = stm32_hash_update,
1952		.base.final = stm32_hash_final,
1953		.base.finup = stm32_hash_finup,
1954		.base.digest = stm32_hash_digest,
1955		.base.export = stm32_hash_export,
1956		.base.import = stm32_hash_import,
1957		.base.halg = {
1958			.digestsize = SHA3_224_DIGEST_SIZE,
1959			.statesize = sizeof(struct stm32_hash_state),
1960			.base = {
1961				.cra_name = "sha3-224",
1962				.cra_driver_name = "stm32-sha3-224",
1963				.cra_priority = 200,
1964				.cra_flags = CRYPTO_ALG_ASYNC |
1965					CRYPTO_ALG_KERN_DRIVER_ONLY,
1966				.cra_blocksize = SHA3_224_BLOCK_SIZE,
1967				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1968				.cra_init = stm32_hash_cra_sha3_init,
1969				.cra_exit = stm32_hash_cra_exit,
1970				.cra_module = THIS_MODULE,
1971			}
1972		},
1973		.op = {
1974			.do_one_request = stm32_hash_one_request,
1975		},
1976	},
1977	{
1978		.base.init = stm32_hash_init,
1979		.base.update = stm32_hash_update,
1980		.base.final = stm32_hash_final,
1981		.base.finup = stm32_hash_finup,
1982		.base.digest = stm32_hash_digest,
1983		.base.export = stm32_hash_export,
1984		.base.import = stm32_hash_import,
1985		.base.setkey = stm32_hash_setkey,
1986		.base.halg = {
1987			.digestsize = SHA3_224_DIGEST_SIZE,
1988			.statesize = sizeof(struct stm32_hash_state),
1989			.base = {
1990				.cra_name = "hmac(sha3-224)",
1991				.cra_driver_name = "stm32-hmac-sha3-224",
1992				.cra_priority = 200,
1993				.cra_flags = CRYPTO_ALG_ASYNC |
1994					CRYPTO_ALG_KERN_DRIVER_ONLY,
1995				.cra_blocksize = SHA3_224_BLOCK_SIZE,
1996				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1997				.cra_init = stm32_hash_cra_sha3_hmac_init,
1998				.cra_exit = stm32_hash_cra_exit,
1999				.cra_module = THIS_MODULE,
2000			}
2001		},
2002		.op = {
2003			.do_one_request = stm32_hash_one_request,
2004		},
2005	},
2006	{
2007		.base.init = stm32_hash_init,
2008		.base.update = stm32_hash_update,
2009		.base.final = stm32_hash_final,
2010		.base.finup = stm32_hash_finup,
2011		.base.digest = stm32_hash_digest,
2012		.base.export = stm32_hash_export,
2013		.base.import = stm32_hash_import,
2014		.base.halg = {
2015			.digestsize = SHA3_256_DIGEST_SIZE,
2016			.statesize = sizeof(struct stm32_hash_state),
2017			.base = {
2018				.cra_name = "sha3-256",
2019				.cra_driver_name = "stm32-sha3-256",
2020				.cra_priority = 200,
2021				.cra_flags = CRYPTO_ALG_ASYNC |
2022					CRYPTO_ALG_KERN_DRIVER_ONLY,
2023				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2024				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2025				.cra_init = stm32_hash_cra_sha3_init,
2026				.cra_exit = stm32_hash_cra_exit,
2027				.cra_module = THIS_MODULE,
2028			}
2029		},
2030		.op = {
2031			.do_one_request = stm32_hash_one_request,
2032		},
2033	},
2034	{
2035		.base.init = stm32_hash_init,
2036		.base.update = stm32_hash_update,
2037		.base.final = stm32_hash_final,
2038		.base.finup = stm32_hash_finup,
2039		.base.digest = stm32_hash_digest,
2040		.base.export = stm32_hash_export,
2041		.base.import = stm32_hash_import,
2042		.base.setkey = stm32_hash_setkey,
2043		.base.halg = {
2044			.digestsize = SHA3_256_DIGEST_SIZE,
2045			.statesize = sizeof(struct stm32_hash_state),
2046			.base = {
2047				.cra_name = "hmac(sha3-256)",
2048				.cra_driver_name = "stm32-hmac-sha3-256",
2049				.cra_priority = 200,
2050				.cra_flags = CRYPTO_ALG_ASYNC |
2051					CRYPTO_ALG_KERN_DRIVER_ONLY,
2052				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2053				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2054				.cra_init = stm32_hash_cra_sha3_hmac_init,
2055				.cra_exit = stm32_hash_cra_exit,
2056				.cra_module = THIS_MODULE,
2057			}
2058		},
2059		.op = {
2060			.do_one_request = stm32_hash_one_request,
2061		},
2062	},
2063	{
2064		.base.init = stm32_hash_init,
2065		.base.update = stm32_hash_update,
2066		.base.final = stm32_hash_final,
2067		.base.finup = stm32_hash_finup,
2068		.base.digest = stm32_hash_digest,
2069		.base.export = stm32_hash_export,
2070		.base.import = stm32_hash_import,
2071		.base.halg = {
2072			.digestsize = SHA3_384_DIGEST_SIZE,
2073			.statesize = sizeof(struct stm32_hash_state),
2074			.base = {
2075				.cra_name = "sha3-384",
2076				.cra_driver_name = "stm32-sha3-384",
2077				.cra_priority = 200,
2078				.cra_flags = CRYPTO_ALG_ASYNC |
2079					CRYPTO_ALG_KERN_DRIVER_ONLY,
2080				.cra_blocksize = SHA3_384_BLOCK_SIZE,
2081				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2082				.cra_init = stm32_hash_cra_sha3_init,
2083				.cra_exit = stm32_hash_cra_exit,
2084				.cra_module = THIS_MODULE,
2085			}
2086		},
2087		.op = {
2088			.do_one_request = stm32_hash_one_request,
2089		},
2090	},
2091	{
2092		.base.init = stm32_hash_init,
2093		.base.update = stm32_hash_update,
2094		.base.final = stm32_hash_final,
2095		.base.finup = stm32_hash_finup,
2096		.base.digest = stm32_hash_digest,
2097		.base.export = stm32_hash_export,
2098		.base.import = stm32_hash_import,
2099		.base.setkey = stm32_hash_setkey,
2100		.base.halg = {
2101			.digestsize = SHA3_384_DIGEST_SIZE,
2102			.statesize = sizeof(struct stm32_hash_state),
2103			.base = {
2104				.cra_name = "hmac(sha3-384)",
2105				.cra_driver_name = "stm32-hmac-sha3-384",
2106				.cra_priority = 200,
2107				.cra_flags = CRYPTO_ALG_ASYNC |
2108					CRYPTO_ALG_KERN_DRIVER_ONLY,
2109				.cra_blocksize = SHA3_384_BLOCK_SIZE,
2110				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2111				.cra_init = stm32_hash_cra_sha3_hmac_init,
2112				.cra_exit = stm32_hash_cra_exit,
2113				.cra_module = THIS_MODULE,
2114			}
2115		},
2116		.op = {
2117			.do_one_request = stm32_hash_one_request,
2118		},
2119	},
2120	{
2121		.base.init = stm32_hash_init,
2122		.base.update = stm32_hash_update,
2123		.base.final = stm32_hash_final,
2124		.base.finup = stm32_hash_finup,
2125		.base.digest = stm32_hash_digest,
2126		.base.export = stm32_hash_export,
2127		.base.import = stm32_hash_import,
2128		.base.halg = {
2129			.digestsize = SHA3_512_DIGEST_SIZE,
2130			.statesize = sizeof(struct stm32_hash_state),
2131			.base = {
2132				.cra_name = "sha3-512",
2133				.cra_driver_name = "stm32-sha3-512",
2134				.cra_priority = 200,
2135				.cra_flags = CRYPTO_ALG_ASYNC |
2136					CRYPTO_ALG_KERN_DRIVER_ONLY,
2137				.cra_blocksize = SHA3_512_BLOCK_SIZE,
2138				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2139				.cra_init = stm32_hash_cra_sha3_init,
2140				.cra_exit = stm32_hash_cra_exit,
2141				.cra_module = THIS_MODULE,
2142			}
2143		},
2144		.op = {
2145			.do_one_request = stm32_hash_one_request,
2146		},
2147	},
2148	{
2149		.base.init = stm32_hash_init,
2150		.base.update = stm32_hash_update,
2151		.base.final = stm32_hash_final,
2152		.base.finup = stm32_hash_finup,
2153		.base.digest = stm32_hash_digest,
2154		.base.export = stm32_hash_export,
2155		.base.import = stm32_hash_import,
2156		.base.setkey = stm32_hash_setkey,
2157		.base.halg = {
2158			.digestsize = SHA3_512_DIGEST_SIZE,
2159			.statesize = sizeof(struct stm32_hash_state),
2160			.base = {
2161				.cra_name = "hmac(sha3-512)",
2162				.cra_driver_name = "stm32-hmac-sha3-512",
2163				.cra_priority = 200,
2164				.cra_flags = CRYPTO_ALG_ASYNC |
2165					CRYPTO_ALG_KERN_DRIVER_ONLY,
2166				.cra_blocksize = SHA3_512_BLOCK_SIZE,
2167				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
2168				.cra_init = stm32_hash_cra_sha3_hmac_init,
2169				.cra_exit = stm32_hash_cra_exit,
2170				.cra_module = THIS_MODULE,
2171			}
2172		},
2173		.op = {
2174			.do_one_request = stm32_hash_one_request,
2175		},
2176	}
2177};
2178
2179static int stm32_hash_register_algs(struct stm32_hash_dev *hdev)
2180{
2181	unsigned int i, j;
2182	int err;
2183
2184	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
2185		for (j = 0; j < hdev->pdata->algs_info[i].size; j++) {
2186			err = crypto_engine_register_ahash(
2187				&hdev->pdata->algs_info[i].algs_list[j]);
2188			if (err)
2189				goto err_algs;
2190		}
2191	}
2192
2193	return 0;
2194err_algs:
2195	dev_err(hdev->dev, "Algo %d : %d failed\n", i, j);
2196	for (; i--; ) {
2197		for (; j--;)
2198			crypto_engine_unregister_ahash(
2199				&hdev->pdata->algs_info[i].algs_list[j]);
2200	}
2201
2202	return err;
2203}
2204
2205static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev)
2206{
2207	unsigned int i, j;
2208
2209	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
2210		for (j = 0; j < hdev->pdata->algs_info[i].size; j++)
2211			crypto_engine_unregister_ahash(
2212				&hdev->pdata->algs_info[i].algs_list[j]);
2213	}
2214
2215	return 0;
2216}
2217
2218static struct stm32_hash_algs_info stm32_hash_algs_info_ux500[] = {
2219	{
2220		.algs_list	= algs_sha1,
2221		.size		= ARRAY_SIZE(algs_sha1),
2222	},
2223	{
2224		.algs_list	= algs_sha256,
2225		.size		= ARRAY_SIZE(algs_sha256),
2226	},
2227};
2228
2229static const struct stm32_hash_pdata stm32_hash_pdata_ux500 = {
2230	.alg_shift	= 7,
2231	.algs_info	= stm32_hash_algs_info_ux500,
2232	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_ux500),
2233	.broken_emptymsg = true,
2234	.ux500		= true,
2235};
2236
2237static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = {
2238	{
2239		.algs_list	= algs_md5,
2240		.size		= ARRAY_SIZE(algs_md5),
2241	},
2242	{
2243		.algs_list	= algs_sha1,
2244		.size		= ARRAY_SIZE(algs_sha1),
2245	},
2246};
2247
2248static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = {
2249	.alg_shift	= 7,
2250	.algs_info	= stm32_hash_algs_info_stm32f4,
2251	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f4),
2252	.has_sr		= true,
2253	.has_mdmat	= true,
2254};
2255
2256static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = {
2257	{
2258		.algs_list	= algs_md5,
2259		.size		= ARRAY_SIZE(algs_md5),
2260	},
2261	{
2262		.algs_list	= algs_sha1,
2263		.size		= ARRAY_SIZE(algs_sha1),
2264	},
2265	{
2266		.algs_list	= algs_sha224,
2267		.size		= ARRAY_SIZE(algs_sha224),
2268	},
2269	{
2270		.algs_list	= algs_sha256,
2271		.size		= ARRAY_SIZE(algs_sha256),
2272	},
2273};
2274
2275static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = {
2276	.alg_shift	= 7,
2277	.algs_info	= stm32_hash_algs_info_stm32f7,
2278	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f7),
2279	.has_sr		= true,
2280	.has_mdmat	= true,
2281};
2282
2283static struct stm32_hash_algs_info stm32_hash_algs_info_stm32mp13[] = {
2284	{
2285		.algs_list	= algs_sha1,
2286		.size		= ARRAY_SIZE(algs_sha1),
2287	},
2288	{
2289		.algs_list	= algs_sha224,
2290		.size		= ARRAY_SIZE(algs_sha224),
2291	},
2292	{
2293		.algs_list	= algs_sha256,
2294		.size		= ARRAY_SIZE(algs_sha256),
2295	},
2296	{
2297		.algs_list	= algs_sha384_sha512,
2298		.size		= ARRAY_SIZE(algs_sha384_sha512),
2299	},
2300	{
2301		.algs_list	= algs_sha3,
2302		.size		= ARRAY_SIZE(algs_sha3),
2303	},
2304};
2305
2306static const struct stm32_hash_pdata stm32_hash_pdata_stm32mp13 = {
2307	.alg_shift	= 17,
2308	.algs_info	= stm32_hash_algs_info_stm32mp13,
2309	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32mp13),
2310	.has_sr		= true,
2311	.has_mdmat	= true,
2312	.context_secured = true,
2313};
2314
2315static const struct of_device_id stm32_hash_of_match[] = {
2316	{ .compatible = "stericsson,ux500-hash", .data = &stm32_hash_pdata_ux500 },
2317	{ .compatible = "st,stm32f456-hash", .data = &stm32_hash_pdata_stm32f4 },
2318	{ .compatible = "st,stm32f756-hash", .data = &stm32_hash_pdata_stm32f7 },
2319	{ .compatible = "st,stm32mp13-hash", .data = &stm32_hash_pdata_stm32mp13 },
2320	{},
2321};
2322
2323MODULE_DEVICE_TABLE(of, stm32_hash_of_match);
2324
2325static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev,
2326				   struct device *dev)
2327{
2328	hdev->pdata = of_device_get_match_data(dev);
2329	if (!hdev->pdata) {
2330		dev_err(dev, "no compatible OF match\n");
2331		return -EINVAL;
2332	}
2333
 
 
 
 
 
 
2334	return 0;
2335}
2336
2337static int stm32_hash_probe(struct platform_device *pdev)
2338{
2339	struct stm32_hash_dev *hdev;
2340	struct device *dev = &pdev->dev;
2341	struct resource *res;
2342	int ret, irq;
2343
2344	hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
2345	if (!hdev)
2346		return -ENOMEM;
2347
2348	hdev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 
2349	if (IS_ERR(hdev->io_base))
2350		return PTR_ERR(hdev->io_base);
2351
2352	hdev->phys_base = res->start;
2353
2354	ret = stm32_hash_get_of_match(hdev, dev);
2355	if (ret)
2356		return ret;
2357
2358	irq = platform_get_irq_optional(pdev, 0);
2359	if (irq < 0 && irq != -ENXIO)
 
2360		return irq;
 
2361
2362	if (irq > 0) {
2363		ret = devm_request_threaded_irq(dev, irq,
2364						stm32_hash_irq_handler,
2365						stm32_hash_irq_thread,
2366						IRQF_ONESHOT,
2367						dev_name(dev), hdev);
2368		if (ret) {
2369			dev_err(dev, "Cannot grab IRQ\n");
2370			return ret;
2371		}
2372	} else {
2373		dev_info(dev, "No IRQ, use polling mode\n");
2374		hdev->polled = true;
2375	}
2376
2377	hdev->clk = devm_clk_get(&pdev->dev, NULL);
2378	if (IS_ERR(hdev->clk))
2379		return dev_err_probe(dev, PTR_ERR(hdev->clk),
2380				     "failed to get clock for hash\n");
 
 
2381
2382	ret = clk_prepare_enable(hdev->clk);
2383	if (ret) {
2384		dev_err(dev, "failed to enable hash clock (%d)\n", ret);
2385		return ret;
2386	}
2387
2388	pm_runtime_set_autosuspend_delay(dev, HASH_AUTOSUSPEND_DELAY);
2389	pm_runtime_use_autosuspend(dev);
2390
2391	pm_runtime_get_noresume(dev);
2392	pm_runtime_set_active(dev);
2393	pm_runtime_enable(dev);
2394
2395	hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
2396	if (IS_ERR(hdev->rst)) {
2397		if (PTR_ERR(hdev->rst) == -EPROBE_DEFER) {
2398			ret = -EPROBE_DEFER;
2399			goto err_reset;
2400		}
2401	} else {
2402		reset_control_assert(hdev->rst);
2403		udelay(2);
2404		reset_control_deassert(hdev->rst);
2405	}
2406
2407	hdev->dev = dev;
2408
2409	platform_set_drvdata(pdev, hdev);
2410
2411	ret = stm32_hash_dma_init(hdev);
2412	switch (ret) {
2413	case 0:
2414		break;
2415	case -ENOENT:
2416	case -ENODEV:
2417		dev_info(dev, "DMA mode not available\n");
2418		break;
2419	default:
2420		dev_err(dev, "DMA init error %d\n", ret);
2421		goto err_dma;
2422	}
2423
2424	spin_lock(&stm32_hash.lock);
2425	list_add_tail(&hdev->list, &stm32_hash.dev_list);
2426	spin_unlock(&stm32_hash.lock);
2427
2428	/* Initialize crypto engine */
2429	hdev->engine = crypto_engine_alloc_init(dev, 1);
2430	if (!hdev->engine) {
2431		ret = -ENOMEM;
2432		goto err_engine;
2433	}
2434
2435	ret = crypto_engine_start(hdev->engine);
2436	if (ret)
2437		goto err_engine_start;
2438
2439	if (hdev->pdata->ux500)
2440		/* FIXME: implement DMA mode for Ux500 */
2441		hdev->dma_mode = 0;
2442	else
2443		hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR) & HASH_HWCFG_DMA_MASK;
2444
2445	/* Register algos */
2446	ret = stm32_hash_register_algs(hdev);
2447	if (ret)
2448		goto err_algs;
2449
2450	dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
2451		 stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
2452
2453	pm_runtime_put_sync(dev);
2454
2455	return 0;
2456
2457err_algs:
2458err_engine_start:
2459	crypto_engine_exit(hdev->engine);
2460err_engine:
2461	spin_lock(&stm32_hash.lock);
2462	list_del(&hdev->list);
2463	spin_unlock(&stm32_hash.lock);
2464err_dma:
2465	if (hdev->dma_lch)
2466		dma_release_channel(hdev->dma_lch);
2467err_reset:
2468	pm_runtime_disable(dev);
2469	pm_runtime_put_noidle(dev);
2470
2471	clk_disable_unprepare(hdev->clk);
2472
2473	return ret;
2474}
2475
2476static void stm32_hash_remove(struct platform_device *pdev)
2477{
2478	struct stm32_hash_dev *hdev = platform_get_drvdata(pdev);
2479	int ret;
2480
2481	ret = pm_runtime_get_sync(hdev->dev);
 
 
2482
2483	stm32_hash_unregister_algs(hdev);
2484
2485	crypto_engine_exit(hdev->engine);
2486
2487	spin_lock(&stm32_hash.lock);
2488	list_del(&hdev->list);
2489	spin_unlock(&stm32_hash.lock);
2490
2491	if (hdev->dma_lch)
2492		dma_release_channel(hdev->dma_lch);
2493
2494	pm_runtime_disable(hdev->dev);
2495	pm_runtime_put_noidle(hdev->dev);
2496
2497	if (ret >= 0)
2498		clk_disable_unprepare(hdev->clk);
2499}
2500
2501#ifdef CONFIG_PM
2502static int stm32_hash_runtime_suspend(struct device *dev)
2503{
2504	struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2505
2506	clk_disable_unprepare(hdev->clk);
2507
2508	return 0;
2509}
2510
2511static int stm32_hash_runtime_resume(struct device *dev)
2512{
2513	struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2514	int ret;
2515
2516	ret = clk_prepare_enable(hdev->clk);
2517	if (ret) {
2518		dev_err(hdev->dev, "Failed to prepare_enable clock\n");
2519		return ret;
2520	}
2521
2522	return 0;
2523}
2524#endif
2525
2526static const struct dev_pm_ops stm32_hash_pm_ops = {
2527	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2528				pm_runtime_force_resume)
2529	SET_RUNTIME_PM_OPS(stm32_hash_runtime_suspend,
2530			   stm32_hash_runtime_resume, NULL)
2531};
2532
2533static struct platform_driver stm32_hash_driver = {
2534	.probe		= stm32_hash_probe,
2535	.remove		= stm32_hash_remove,
2536	.driver		= {
2537		.name	= "stm32-hash",
2538		.pm = &stm32_hash_pm_ops,
2539		.of_match_table	= stm32_hash_of_match,
2540	}
2541};
2542
2543module_platform_driver(stm32_hash_driver);
2544
2545MODULE_DESCRIPTION("STM32 SHA1/SHA2/SHA3 & MD5 (HMAC) hw accelerator driver");
2546MODULE_AUTHOR("Lionel Debieve <lionel.debieve@st.com>");
2547MODULE_LICENSE("GPL v2");