1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
   4 * All rights reserved.
   5 */
   6
   7#include <linux/kernel.h>
   8#include <linux/module.h>
   9#include <linux/moduleparam.h>
  10#include <linux/mod_devicetable.h>
  11#include <linux/interrupt.h>
  12#include <linux/pci.h>
  13#include <linux/slab.h>
  14#include <linux/delay.h>
  15#include <linux/mm.h>
  16#include <linux/dma-mapping.h>
  17#include <linux/scatterlist.h>
  18#include <linux/highmem.h>
  19#include <linux/crypto.h>
  20#include <linux/hw_random.h>
  21#include <linux/ktime.h>
  22
  23#include <crypto/algapi.h>
  24#include <crypto/internal/des.h>
  25#include <crypto/internal/skcipher.h>
  26
  27static char hifn_pll_ref[sizeof("extNNN")] = "ext";
  28module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
  29MODULE_PARM_DESC(hifn_pll_ref,
  30		 "PLL reference clock (pci[freq] or ext[freq], default ext)");
  31
  32static atomic_t hifn_dev_number;
  33
  34#define ACRYPTO_OP_DECRYPT	0
  35#define ACRYPTO_OP_ENCRYPT	1
  36#define ACRYPTO_OP_HMAC		2
  37#define ACRYPTO_OP_RNG		3
  38
  39#define ACRYPTO_MODE_ECB		0
  40#define ACRYPTO_MODE_CBC		1
  41#define ACRYPTO_MODE_CFB		2
  42#define ACRYPTO_MODE_OFB		3
  43
  44#define ACRYPTO_TYPE_AES_128	0
  45#define ACRYPTO_TYPE_AES_192	1
  46#define ACRYPTO_TYPE_AES_256	2
  47#define ACRYPTO_TYPE_3DES	3
  48#define ACRYPTO_TYPE_DES	4
  49
  50#define PCI_VENDOR_ID_HIFN		0x13A3
  51#define PCI_DEVICE_ID_HIFN_7955		0x0020
  52#define	PCI_DEVICE_ID_HIFN_7956		0x001d
  53
  54/* I/O region sizes */
  55
  56#define HIFN_BAR0_SIZE			0x1000
  57#define HIFN_BAR1_SIZE			0x2000
  58#define HIFN_BAR2_SIZE			0x8000
  59
  60/* DMA registres */
  61
  62#define HIFN_DMA_CRA			0x0C	/* DMA Command Ring Address */
  63#define HIFN_DMA_SDRA			0x1C	/* DMA Source Data Ring Address */
  64#define HIFN_DMA_RRA			0x2C	/* DMA Result Ring Address */
  65#define HIFN_DMA_DDRA			0x3C	/* DMA Destination Data Ring Address */
  66#define HIFN_DMA_STCTL			0x40	/* DMA Status and Control */
  67#define HIFN_DMA_INTREN			0x44	/* DMA Interrupt Enable */
  68#define HIFN_DMA_CFG1			0x48	/* DMA Configuration #1 */
  69#define HIFN_DMA_CFG2			0x6C	/* DMA Configuration #2 */
  70#define HIFN_CHIP_ID			0x98	/* Chip ID */
  71
  72/*
  73 * Processing Unit Registers (offset from BASEREG0)
  74 */
  75#define	HIFN_0_PUDATA		0x00	/* Processing Unit Data */
  76#define	HIFN_0_PUCTRL		0x04	/* Processing Unit Control */
  77#define	HIFN_0_PUISR		0x08	/* Processing Unit Interrupt Status */
  78#define	HIFN_0_PUCNFG		0x0c	/* Processing Unit Configuration */
  79#define	HIFN_0_PUIER		0x10	/* Processing Unit Interrupt Enable */
  80#define	HIFN_0_PUSTAT		0x14	/* Processing Unit Status/Chip ID */
  81#define	HIFN_0_FIFOSTAT		0x18	/* FIFO Status */
  82#define	HIFN_0_FIFOCNFG		0x1c	/* FIFO Configuration */
  83#define	HIFN_0_SPACESIZE	0x20	/* Register space size */
  84
  85/* Processing Unit Control Register (HIFN_0_PUCTRL) */
  86#define	HIFN_PUCTRL_CLRSRCFIFO	0x0010	/* clear source fifo */
  87#define	HIFN_PUCTRL_STOP	0x0008	/* stop pu */
  88#define	HIFN_PUCTRL_LOCKRAM	0x0004	/* lock ram */
  89#define	HIFN_PUCTRL_DMAENA	0x0002	/* enable dma */
  90#define	HIFN_PUCTRL_RESET	0x0001	/* Reset processing unit */
  91
  92/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
  93#define	HIFN_PUISR_CMDINVAL	0x8000	/* Invalid command interrupt */
  94#define	HIFN_PUISR_DATAERR	0x4000	/* Data error interrupt */
  95#define	HIFN_PUISR_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
  96#define	HIFN_PUISR_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
  97#define	HIFN_PUISR_DSTOVER	0x0200	/* Destination overrun interrupt */
  98#define	HIFN_PUISR_SRCCMD	0x0080	/* Source command interrupt */
  99#define	HIFN_PUISR_SRCCTX	0x0040	/* Source context interrupt */
 100#define	HIFN_PUISR_SRCDATA	0x0020	/* Source data interrupt */
 101#define	HIFN_PUISR_DSTDATA	0x0010	/* Destination data interrupt */
 102#define	HIFN_PUISR_DSTRESULT	0x0004	/* Destination result interrupt */
 103
 104/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
 105#define	HIFN_PUCNFG_DRAMMASK	0xe000	/* DRAM size mask */
 106#define	HIFN_PUCNFG_DSZ_256K	0x0000	/* 256k dram */
 107#define	HIFN_PUCNFG_DSZ_512K	0x2000	/* 512k dram */
 108#define	HIFN_PUCNFG_DSZ_1M	0x4000	/* 1m dram */
 109#define	HIFN_PUCNFG_DSZ_2M	0x6000	/* 2m dram */
 110#define	HIFN_PUCNFG_DSZ_4M	0x8000	/* 4m dram */
 111#define	HIFN_PUCNFG_DSZ_8M	0xa000	/* 8m dram */
 112#define	HIFN_PUNCFG_DSZ_16M	0xc000	/* 16m dram */
 113#define	HIFN_PUCNFG_DSZ_32M	0xe000	/* 32m dram */
 114#define	HIFN_PUCNFG_DRAMREFRESH	0x1800	/* DRAM refresh rate mask */
 115#define	HIFN_PUCNFG_DRFR_512	0x0000	/* 512 divisor of ECLK */
 116#define	HIFN_PUCNFG_DRFR_256	0x0800	/* 256 divisor of ECLK */
 117#define	HIFN_PUCNFG_DRFR_128	0x1000	/* 128 divisor of ECLK */
 118#define	HIFN_PUCNFG_TCALLPHASES	0x0200	/* your guess is as good as mine... */
 119#define	HIFN_PUCNFG_TCDRVTOTEM	0x0100	/* your guess is as good as mine... */
 120#define	HIFN_PUCNFG_BIGENDIAN	0x0080	/* DMA big endian mode */
 121#define	HIFN_PUCNFG_BUS32	0x0040	/* Bus width 32bits */
 122#define	HIFN_PUCNFG_BUS16	0x0000	/* Bus width 16 bits */
 123#define	HIFN_PUCNFG_CHIPID	0x0020	/* Allow chipid from PUSTAT */
 124#define	HIFN_PUCNFG_DRAM	0x0010	/* Context RAM is DRAM */
 125#define	HIFN_PUCNFG_SRAM	0x0000	/* Context RAM is SRAM */
 126#define	HIFN_PUCNFG_COMPSING	0x0004	/* Enable single compression context */
 127#define	HIFN_PUCNFG_ENCCNFG	0x0002	/* Encryption configuration */
 128
 129/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
 130#define	HIFN_PUIER_CMDINVAL	0x8000	/* Invalid command interrupt */
 131#define	HIFN_PUIER_DATAERR	0x4000	/* Data error interrupt */
 132#define	HIFN_PUIER_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
 133#define	HIFN_PUIER_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
 134#define	HIFN_PUIER_DSTOVER	0x0200	/* Destination overrun interrupt */
 135#define	HIFN_PUIER_SRCCMD	0x0080	/* Source command interrupt */
 136#define	HIFN_PUIER_SRCCTX	0x0040	/* Source context interrupt */
 137#define	HIFN_PUIER_SRCDATA	0x0020	/* Source data interrupt */
 138#define	HIFN_PUIER_DSTDATA	0x0010	/* Destination data interrupt */
 139#define	HIFN_PUIER_DSTRESULT	0x0004	/* Destination result interrupt */
 140
 141/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
 142#define	HIFN_PUSTAT_CMDINVAL	0x8000	/* Invalid command interrupt */
 143#define	HIFN_PUSTAT_DATAERR	0x4000	/* Data error interrupt */
 144#define	HIFN_PUSTAT_SRCFIFO	0x2000	/* Source FIFO ready interrupt */
 145#define	HIFN_PUSTAT_DSTFIFO	0x1000	/* Destination FIFO ready interrupt */
 146#define	HIFN_PUSTAT_DSTOVER	0x0200	/* Destination overrun interrupt */
 147#define	HIFN_PUSTAT_SRCCMD	0x0080	/* Source command interrupt */
 148#define	HIFN_PUSTAT_SRCCTX	0x0040	/* Source context interrupt */
 149#define	HIFN_PUSTAT_SRCDATA	0x0020	/* Source data interrupt */
 150#define	HIFN_PUSTAT_DSTDATA	0x0010	/* Destination data interrupt */
 151#define	HIFN_PUSTAT_DSTRESULT	0x0004	/* Destination result interrupt */
 152#define	HIFN_PUSTAT_CHIPREV	0x00ff	/* Chip revision mask */
 153#define	HIFN_PUSTAT_CHIPENA	0xff00	/* Chip enabled mask */
 154#define	HIFN_PUSTAT_ENA_2	0x1100	/* Level 2 enabled */
 155#define	HIFN_PUSTAT_ENA_1	0x1000	/* Level 1 enabled */
 156#define	HIFN_PUSTAT_ENA_0	0x3000	/* Level 0 enabled */
 157#define	HIFN_PUSTAT_REV_2	0x0020	/* 7751 PT6/2 */
 158#define	HIFN_PUSTAT_REV_3	0x0030	/* 7751 PT6/3 */
 159
 160/* FIFO Status Register (HIFN_0_FIFOSTAT) */
 161#define	HIFN_FIFOSTAT_SRC	0x7f00	/* Source FIFO available */
 162#define	HIFN_FIFOSTAT_DST	0x007f	/* Destination FIFO available */
 163
 164/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
 165#define	HIFN_FIFOCNFG_THRESHOLD	0x0400	/* must be written as 1 */
 166
 167/*
 168 * DMA Interface Registers (offset from BASEREG1)
 169 */
 170#define	HIFN_1_DMA_CRAR		0x0c	/* DMA Command Ring Address */
 171#define	HIFN_1_DMA_SRAR		0x1c	/* DMA Source Ring Address */
 172#define	HIFN_1_DMA_RRAR		0x2c	/* DMA Result Ring Address */
 173#define	HIFN_1_DMA_DRAR		0x3c	/* DMA Destination Ring Address */
 174#define	HIFN_1_DMA_CSR		0x40	/* DMA Status and Control */
 175#define	HIFN_1_DMA_IER		0x44	/* DMA Interrupt Enable */
 176#define	HIFN_1_DMA_CNFG		0x48	/* DMA Configuration */
 177#define	HIFN_1_PLL		0x4c	/* 795x: PLL config */
 178#define	HIFN_1_7811_RNGENA	0x60	/* 7811: rng enable */
 179#define	HIFN_1_7811_RNGCFG	0x64	/* 7811: rng config */
 180#define	HIFN_1_7811_RNGDAT	0x68	/* 7811: rng data */
 181#define	HIFN_1_7811_RNGSTS	0x6c	/* 7811: rng status */
 182#define	HIFN_1_7811_MIPSRST	0x94	/* 7811: MIPS reset */
 183#define	HIFN_1_REVID		0x98	/* Revision ID */
 184#define	HIFN_1_UNLOCK_SECRET1	0xf4
 185#define	HIFN_1_UNLOCK_SECRET2	0xfc
 186#define	HIFN_1_PUB_RESET	0x204	/* Public/RNG Reset */
 187#define	HIFN_1_PUB_BASE		0x300	/* Public Base Address */
 188#define	HIFN_1_PUB_OPLEN	0x304	/* Public Operand Length */
 189#define	HIFN_1_PUB_OP		0x308	/* Public Operand */
 190#define	HIFN_1_PUB_STATUS	0x30c	/* Public Status */
 191#define	HIFN_1_PUB_IEN		0x310	/* Public Interrupt enable */
 192#define	HIFN_1_RNG_CONFIG	0x314	/* RNG config */
 193#define	HIFN_1_RNG_DATA		0x318	/* RNG data */
 194#define	HIFN_1_PUB_MEM		0x400	/* start of Public key memory */
 195#define	HIFN_1_PUB_MEMEND	0xbff	/* end of Public key memory */
 196
 197/* DMA Status and Control Register (HIFN_1_DMA_CSR) */
 198#define	HIFN_DMACSR_D_CTRLMASK	0xc0000000	/* Destinition Ring Control */
 199#define	HIFN_DMACSR_D_CTRL_NOP	0x00000000	/* Dest. Control: no-op */
 200#define	HIFN_DMACSR_D_CTRL_DIS	0x40000000	/* Dest. Control: disable */
 201#define	HIFN_DMACSR_D_CTRL_ENA	0x80000000	/* Dest. Control: enable */
 202#define	HIFN_DMACSR_D_ABORT	0x20000000	/* Destinition Ring PCIAbort */
 203#define	HIFN_DMACSR_D_DONE	0x10000000	/* Destinition Ring Done */
 204#define	HIFN_DMACSR_D_LAST	0x08000000	/* Destinition Ring Last */
 205#define	HIFN_DMACSR_D_WAIT	0x04000000	/* Destinition Ring Waiting */
 206#define	HIFN_DMACSR_D_OVER	0x02000000	/* Destinition Ring Overflow */
 207#define	HIFN_DMACSR_R_CTRL	0x00c00000	/* Result Ring Control */
 208#define	HIFN_DMACSR_R_CTRL_NOP	0x00000000	/* Result Control: no-op */
 209#define	HIFN_DMACSR_R_CTRL_DIS	0x00400000	/* Result Control: disable */
 210#define	HIFN_DMACSR_R_CTRL_ENA	0x00800000	/* Result Control: enable */
 211#define	HIFN_DMACSR_R_ABORT	0x00200000	/* Result Ring PCI Abort */
 212#define	HIFN_DMACSR_R_DONE	0x00100000	/* Result Ring Done */
 213#define	HIFN_DMACSR_R_LAST	0x00080000	/* Result Ring Last */
 214#define	HIFN_DMACSR_R_WAIT	0x00040000	/* Result Ring Waiting */
 215#define	HIFN_DMACSR_R_OVER	0x00020000	/* Result Ring Overflow */
 216#define	HIFN_DMACSR_S_CTRL	0x0000c000	/* Source Ring Control */
 217#define	HIFN_DMACSR_S_CTRL_NOP	0x00000000	/* Source Control: no-op */
 218#define	HIFN_DMACSR_S_CTRL_DIS	0x00004000	/* Source Control: disable */
 219#define	HIFN_DMACSR_S_CTRL_ENA	0x00008000	/* Source Control: enable */
 220#define	HIFN_DMACSR_S_ABORT	0x00002000	/* Source Ring PCI Abort */
 221#define	HIFN_DMACSR_S_DONE	0x00001000	/* Source Ring Done */
 222#define	HIFN_DMACSR_S_LAST	0x00000800	/* Source Ring Last */
 223#define	HIFN_DMACSR_S_WAIT	0x00000400	/* Source Ring Waiting */
 224#define	HIFN_DMACSR_ILLW	0x00000200	/* Illegal write (7811 only) */
 225#define	HIFN_DMACSR_ILLR	0x00000100	/* Illegal read (7811 only) */
 226#define	HIFN_DMACSR_C_CTRL	0x000000c0	/* Command Ring Control */
 227#define	HIFN_DMACSR_C_CTRL_NOP	0x00000000	/* Command Control: no-op */
 228#define	HIFN_DMACSR_C_CTRL_DIS	0x00000040	/* Command Control: disable */
 229#define	HIFN_DMACSR_C_CTRL_ENA	0x00000080	/* Command Control: enable */
 230#define	HIFN_DMACSR_C_ABORT	0x00000020	/* Command Ring PCI Abort */
 231#define	HIFN_DMACSR_C_DONE	0x00000010	/* Command Ring Done */
 232#define	HIFN_DMACSR_C_LAST	0x00000008	/* Command Ring Last */
 233#define	HIFN_DMACSR_C_WAIT	0x00000004	/* Command Ring Waiting */
 234#define	HIFN_DMACSR_PUBDONE	0x00000002	/* Public op done (7951 only) */
 235#define	HIFN_DMACSR_ENGINE	0x00000001	/* Command Ring Engine IRQ */
 236
 237/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
 238#define	HIFN_DMAIER_D_ABORT	0x20000000	/* Destination Ring PCIAbort */
 239#define	HIFN_DMAIER_D_DONE	0x10000000	/* Destination Ring Done */
 240#define	HIFN_DMAIER_D_LAST	0x08000000	/* Destination Ring Last */
 241#define	HIFN_DMAIER_D_WAIT	0x04000000	/* Destination Ring Waiting */
 242#define	HIFN_DMAIER_D_OVER	0x02000000	/* Destination Ring Overflow */
 243#define	HIFN_DMAIER_R_ABORT	0x00200000	/* Result Ring PCI Abort */
 244#define	HIFN_DMAIER_R_DONE	0x00100000	/* Result Ring Done */
 245#define	HIFN_DMAIER_R_LAST	0x00080000	/* Result Ring Last */
 246#define	HIFN_DMAIER_R_WAIT	0x00040000	/* Result Ring Waiting */
 247#define	HIFN_DMAIER_R_OVER	0x00020000	/* Result Ring Overflow */
 248#define	HIFN_DMAIER_S_ABORT	0x00002000	/* Source Ring PCI Abort */
 249#define	HIFN_DMAIER_S_DONE	0x00001000	/* Source Ring Done */
 250#define	HIFN_DMAIER_S_LAST	0x00000800	/* Source Ring Last */
 251#define	HIFN_DMAIER_S_WAIT	0x00000400	/* Source Ring Waiting */
 252#define	HIFN_DMAIER_ILLW	0x00000200	/* Illegal write (7811 only) */
 253#define	HIFN_DMAIER_ILLR	0x00000100	/* Illegal read (7811 only) */
 254#define	HIFN_DMAIER_C_ABORT	0x00000020	/* Command Ring PCI Abort */
 255#define	HIFN_DMAIER_C_DONE	0x00000010	/* Command Ring Done */
 256#define	HIFN_DMAIER_C_LAST	0x00000008	/* Command Ring Last */
 257#define	HIFN_DMAIER_C_WAIT	0x00000004	/* Command Ring Waiting */
 258#define	HIFN_DMAIER_PUBDONE	0x00000002	/* public op done (7951 only) */
 259#define	HIFN_DMAIER_ENGINE	0x00000001	/* Engine IRQ */
 260
 261/* DMA Configuration Register (HIFN_1_DMA_CNFG) */
 262#define	HIFN_DMACNFG_BIGENDIAN	0x10000000	/* big endian mode */
 263#define	HIFN_DMACNFG_POLLFREQ	0x00ff0000	/* Poll frequency mask */
 264#define	HIFN_DMACNFG_UNLOCK	0x00000800
 265#define	HIFN_DMACNFG_POLLINVAL	0x00000700	/* Invalid Poll Scalar */
 266#define	HIFN_DMACNFG_LAST	0x00000010	/* Host control LAST bit */
 267#define	HIFN_DMACNFG_MODE	0x00000004	/* DMA mode */
 268#define	HIFN_DMACNFG_DMARESET	0x00000002	/* DMA Reset # */
 269#define	HIFN_DMACNFG_MSTRESET	0x00000001	/* Master Reset # */
 270
 271/* PLL configuration register */
 272#define HIFN_PLL_REF_CLK_HBI	0x00000000	/* HBI reference clock */
 273#define HIFN_PLL_REF_CLK_PLL	0x00000001	/* PLL reference clock */
 274#define HIFN_PLL_BP		0x00000002	/* Reference clock bypass */
 275#define HIFN_PLL_PK_CLK_HBI	0x00000000	/* PK engine HBI clock */
 276#define HIFN_PLL_PK_CLK_PLL	0x00000008	/* PK engine PLL clock */
 277#define HIFN_PLL_PE_CLK_HBI	0x00000000	/* PE engine HBI clock */
 278#define HIFN_PLL_PE_CLK_PLL	0x00000010	/* PE engine PLL clock */
 279#define HIFN_PLL_RESERVED_1	0x00000400	/* Reserved bit, must be 1 */
 280#define HIFN_PLL_ND_SHIFT	11		/* Clock multiplier shift */
 281#define HIFN_PLL_ND_MULT_2	0x00000000	/* PLL clock multiplier 2 */
 282#define HIFN_PLL_ND_MULT_4	0x00000800	/* PLL clock multiplier 4 */
 283#define HIFN_PLL_ND_MULT_6	0x00001000	/* PLL clock multiplier 6 */
 284#define HIFN_PLL_ND_MULT_8	0x00001800	/* PLL clock multiplier 8 */
 285#define HIFN_PLL_ND_MULT_10	0x00002000	/* PLL clock multiplier 10 */
 286#define HIFN_PLL_ND_MULT_12	0x00002800	/* PLL clock multiplier 12 */
 287#define HIFN_PLL_IS_1_8		0x00000000	/* charge pump (mult. 1-8) */
 288#define HIFN_PLL_IS_9_12	0x00010000	/* charge pump (mult. 9-12) */
 289
 290#define HIFN_PLL_FCK_MAX	266		/* Maximum PLL frequency */
 291
 292/* Public key reset register (HIFN_1_PUB_RESET) */
 293#define	HIFN_PUBRST_RESET	0x00000001	/* reset public/rng unit */
 294
 295/* Public base address register (HIFN_1_PUB_BASE) */
 296#define	HIFN_PUBBASE_ADDR	0x00003fff	/* base address */
 297
 298/* Public operand length register (HIFN_1_PUB_OPLEN) */
 299#define	HIFN_PUBOPLEN_MOD_M	0x0000007f	/* modulus length mask */
 300#define	HIFN_PUBOPLEN_MOD_S	0		/* modulus length shift */
 301#define	HIFN_PUBOPLEN_EXP_M	0x0003ff80	/* exponent length mask */
 302#define	HIFN_PUBOPLEN_EXP_S	7		/* exponent length shift */
 303#define	HIFN_PUBOPLEN_RED_M	0x003c0000	/* reducend length mask */
 304#define	HIFN_PUBOPLEN_RED_S	18		/* reducend length shift */
 305
 306/* Public operation register (HIFN_1_PUB_OP) */
 307#define	HIFN_PUBOP_AOFFSET_M	0x0000007f	/* A offset mask */
 308#define	HIFN_PUBOP_AOFFSET_S	0		/* A offset shift */
 309#define	HIFN_PUBOP_BOFFSET_M	0x00000f80	/* B offset mask */
 310#define	HIFN_PUBOP_BOFFSET_S	7		/* B offset shift */
 311#define	HIFN_PUBOP_MOFFSET_M	0x0003f000	/* M offset mask */
 312#define	HIFN_PUBOP_MOFFSET_S	12		/* M offset shift */
 313#define	HIFN_PUBOP_OP_MASK	0x003c0000	/* Opcode: */
 314#define	HIFN_PUBOP_OP_NOP	0x00000000	/*  NOP */
 315#define	HIFN_PUBOP_OP_ADD	0x00040000	/*  ADD */
 316#define	HIFN_PUBOP_OP_ADDC	0x00080000	/*  ADD w/carry */
 317#define	HIFN_PUBOP_OP_SUB	0x000c0000	/*  SUB */
 318#define	HIFN_PUBOP_OP_SUBC	0x00100000	/*  SUB w/carry */
 319#define	HIFN_PUBOP_OP_MODADD	0x00140000	/*  Modular ADD */
 320#define	HIFN_PUBOP_OP_MODSUB	0x00180000	/*  Modular SUB */
 321#define	HIFN_PUBOP_OP_INCA	0x001c0000	/*  INC A */
 322#define	HIFN_PUBOP_OP_DECA	0x00200000	/*  DEC A */
 323#define	HIFN_PUBOP_OP_MULT	0x00240000	/*  MULT */
 324#define	HIFN_PUBOP_OP_MODMULT	0x00280000	/*  Modular MULT */
 325#define	HIFN_PUBOP_OP_MODRED	0x002c0000	/*  Modular RED */
 326#define	HIFN_PUBOP_OP_MODEXP	0x00300000	/*  Modular EXP */
 327
 328/* Public status register (HIFN_1_PUB_STATUS) */
 329#define	HIFN_PUBSTS_DONE	0x00000001	/* operation done */
 330#define	HIFN_PUBSTS_CARRY	0x00000002	/* carry */
 331
 332/* Public interrupt enable register (HIFN_1_PUB_IEN) */
 333#define	HIFN_PUBIEN_DONE	0x00000001	/* operation done interrupt */
 334
 335/* Random number generator config register (HIFN_1_RNG_CONFIG) */
 336#define	HIFN_RNGCFG_ENA		0x00000001	/* enable rng */
 337
 338#define HIFN_NAMESIZE			32
 339#define HIFN_MAX_RESULT_ORDER		5
 340
 341#define	HIFN_D_CMD_RSIZE		(24 * 1)
 342#define	HIFN_D_SRC_RSIZE		(80 * 1)
 343#define	HIFN_D_DST_RSIZE		(80 * 1)
 344#define	HIFN_D_RES_RSIZE		(24 * 1)
 345
 346#define HIFN_D_DST_DALIGN		4
 347
 348#define HIFN_QUEUE_LENGTH		(HIFN_D_CMD_RSIZE - 1)
 349
 350#define AES_MIN_KEY_SIZE		16
 351#define AES_MAX_KEY_SIZE		32
 352
 353#define HIFN_DES_KEY_LENGTH		8
 354#define HIFN_3DES_KEY_LENGTH		24
 355#define HIFN_MAX_CRYPT_KEY_LENGTH	AES_MAX_KEY_SIZE
 356#define HIFN_IV_LENGTH			8
 357#define HIFN_AES_IV_LENGTH		16
 358#define	HIFN_MAX_IV_LENGTH		HIFN_AES_IV_LENGTH
 359
 360#define HIFN_MAC_KEY_LENGTH		64
 361#define HIFN_MD5_LENGTH			16
 362#define HIFN_SHA1_LENGTH		20
 363#define HIFN_MAC_TRUNC_LENGTH		12
 364
 365#define	HIFN_MAX_COMMAND		(8 + 8 + 8 + 64 + 260)
 366#define	HIFN_MAX_RESULT			(8 + 4 + 4 + 20 + 4)
 367#define HIFN_USED_RESULT		12
 368
 369struct hifn_desc {
 370	volatile __le32		l;
 371	volatile __le32		p;
 372};
 373
 374struct hifn_dma {
 375	struct hifn_desc	cmdr[HIFN_D_CMD_RSIZE + 1];
 376	struct hifn_desc	srcr[HIFN_D_SRC_RSIZE + 1];
 377	struct hifn_desc	dstr[HIFN_D_DST_RSIZE + 1];
 378	struct hifn_desc	resr[HIFN_D_RES_RSIZE + 1];
 379
 380	u8			command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
 381	u8			result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
 382
 383	/*
 384	 *  Our current positions for insertion and removal from the descriptor
 385	 *  rings.
 386	 */
 387	volatile int		cmdi, srci, dsti, resi;
 388	volatile int		cmdu, srcu, dstu, resu;
 389	int			cmdk, srck, dstk, resk;
 390};
 391
 392#define HIFN_FLAG_CMD_BUSY	(1 << 0)
 393#define HIFN_FLAG_SRC_BUSY	(1 << 1)
 394#define HIFN_FLAG_DST_BUSY	(1 << 2)
 395#define HIFN_FLAG_RES_BUSY	(1 << 3)
 396#define HIFN_FLAG_OLD_KEY	(1 << 4)
 397
 398#define HIFN_DEFAULT_ACTIVE_NUM	5
 399
 400struct hifn_device {
 401	char			name[HIFN_NAMESIZE];
 402
 403	int			irq;
 404
 405	struct pci_dev		*pdev;
 406	void __iomem		*bar[3];
 407
 408	void			*desc_virt;
 409	dma_addr_t		desc_dma;
 410
 411	u32			dmareg;
 412
 413	void			*sa[HIFN_D_RES_RSIZE];
 414
 415	spinlock_t		lock;
 416
 417	u32			flags;
 418	int			active, started;
 419	struct delayed_work	work;
 420	unsigned long		reset;
 421	unsigned long		success;
 422	unsigned long		prev_success;
 423
 424	u8			snum;
 425
 426	struct tasklet_struct	tasklet;
 427
 428	struct crypto_queue	queue;
 429	struct list_head	alg_list;
 430
 431	unsigned int		pk_clk_freq;
 432
 433#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 434	unsigned int		rng_wait_time;
 435	ktime_t			rngtime;
 436	struct hwrng		rng;
 437#endif
 438};
 439
 440#define	HIFN_D_LENGTH			0x0000ffff
 441#define	HIFN_D_NOINVALID		0x01000000
 442#define	HIFN_D_MASKDONEIRQ		0x02000000
 443#define	HIFN_D_DESTOVER			0x04000000
 444#define	HIFN_D_OVER			0x08000000
 445#define	HIFN_D_LAST			0x20000000
 446#define	HIFN_D_JUMP			0x40000000
 447#define	HIFN_D_VALID			0x80000000
 448
 449struct hifn_base_command {
 450	volatile __le16		masks;
 451	volatile __le16		session_num;
 452	volatile __le16		total_source_count;
 453	volatile __le16		total_dest_count;
 454};
 455
 456#define	HIFN_BASE_CMD_COMP		0x0100	/* enable compression engine */
 457#define	HIFN_BASE_CMD_PAD		0x0200	/* enable padding engine */
 458#define	HIFN_BASE_CMD_MAC		0x0400	/* enable MAC engine */
 459#define	HIFN_BASE_CMD_CRYPT		0x0800	/* enable crypt engine */
 460#define	HIFN_BASE_CMD_DECODE		0x2000
 461#define	HIFN_BASE_CMD_SRCLEN_M		0xc000
 462#define	HIFN_BASE_CMD_SRCLEN_S		14
 463#define	HIFN_BASE_CMD_DSTLEN_M		0x3000
 464#define	HIFN_BASE_CMD_DSTLEN_S		12
 465#define	HIFN_BASE_CMD_LENMASK_HI	0x30000
 466#define	HIFN_BASE_CMD_LENMASK_LO	0x0ffff
 467
 468/*
 469 * Structure to help build up the command data structure.
 470 */
 471struct hifn_crypt_command {
 472	volatile __le16		masks;
 473	volatile __le16		header_skip;
 474	volatile __le16		source_count;
 475	volatile __le16		reserved;
 476};
 477
 478#define	HIFN_CRYPT_CMD_ALG_MASK		0x0003		/* algorithm: */
 479#define	HIFN_CRYPT_CMD_ALG_DES		0x0000		/*   DES */
 480#define	HIFN_CRYPT_CMD_ALG_3DES		0x0001		/*   3DES */
 481#define	HIFN_CRYPT_CMD_ALG_RC4		0x0002		/*   RC4 */
 482#define	HIFN_CRYPT_CMD_ALG_AES		0x0003		/*   AES */
 483#define	HIFN_CRYPT_CMD_MODE_MASK	0x0018		/* Encrypt mode: */
 484#define	HIFN_CRYPT_CMD_MODE_ECB		0x0000		/*   ECB */
 485#define	HIFN_CRYPT_CMD_MODE_CBC		0x0008		/*   CBC */
 486#define	HIFN_CRYPT_CMD_MODE_CFB		0x0010		/*   CFB */
 487#define	HIFN_CRYPT_CMD_MODE_OFB		0x0018		/*   OFB */
 488#define	HIFN_CRYPT_CMD_CLR_CTX		0x0040		/* clear context */
 489#define	HIFN_CRYPT_CMD_KSZ_MASK		0x0600		/* AES key size: */
 490#define	HIFN_CRYPT_CMD_KSZ_128		0x0000		/*  128 bit */
 491#define	HIFN_CRYPT_CMD_KSZ_192		0x0200		/*  192 bit */
 492#define	HIFN_CRYPT_CMD_KSZ_256		0x0400		/*  256 bit */
 493#define	HIFN_CRYPT_CMD_NEW_KEY		0x0800		/* expect new key */
 494#define	HIFN_CRYPT_CMD_NEW_IV		0x1000		/* expect new iv */
 495#define	HIFN_CRYPT_CMD_SRCLEN_M		0xc000
 496#define	HIFN_CRYPT_CMD_SRCLEN_S		14
 497
 
 
 
 
 
 
 
 
 
 
 498#define	HIFN_MAC_CMD_ALG_MASK		0x0001
 499#define	HIFN_MAC_CMD_ALG_SHA1		0x0000
 500#define	HIFN_MAC_CMD_ALG_MD5		0x0001
 501#define	HIFN_MAC_CMD_MODE_MASK		0x000c
 502#define	HIFN_MAC_CMD_MODE_HMAC		0x0000
 503#define	HIFN_MAC_CMD_MODE_SSL_MAC	0x0004
 504#define	HIFN_MAC_CMD_MODE_HASH		0x0008
 505#define	HIFN_MAC_CMD_MODE_FULL		0x0004
 506#define	HIFN_MAC_CMD_TRUNC		0x0010
 507#define	HIFN_MAC_CMD_RESULT		0x0020
 508#define	HIFN_MAC_CMD_APPEND		0x0040
 509#define	HIFN_MAC_CMD_SRCLEN_M		0xc000
 510#define	HIFN_MAC_CMD_SRCLEN_S		14
 511
 512/*
 513 * MAC POS IPsec initiates authentication after encryption on encodes
 514 * and before decryption on decodes.
 515 */
 516#define	HIFN_MAC_CMD_POS_IPSEC		0x0200
 517#define	HIFN_MAC_CMD_NEW_KEY		0x0800
 518
 
 
 
 
 
 
 
 519#define	HIFN_COMP_CMD_SRCLEN_M		0xc000
 520#define	HIFN_COMP_CMD_SRCLEN_S		14
 521#define	HIFN_COMP_CMD_ONE		0x0100	/* must be one */
 522#define	HIFN_COMP_CMD_CLEARHIST		0x0010	/* clear history */
 523#define	HIFN_COMP_CMD_UPDATEHIST	0x0008	/* update history */
 524#define	HIFN_COMP_CMD_LZS_STRIP0	0x0004	/* LZS: strip zero */
 525#define	HIFN_COMP_CMD_MPPC_RESTART	0x0004	/* MPPC: restart */
 526#define	HIFN_COMP_CMD_ALG_MASK		0x0001	/* compression mode: */
 527#define	HIFN_COMP_CMD_ALG_MPPC		0x0001	/*   MPPC */
 528#define	HIFN_COMP_CMD_ALG_LZS		0x0000	/*   LZS */
 529
 530struct hifn_base_result {
 531	volatile __le16		flags;
 532	volatile __le16		session;
 533	volatile __le16		src_cnt;		/* 15:0 of source count */
 534	volatile __le16		dst_cnt;		/* 15:0 of dest count */
 535};
 536
 537#define	HIFN_BASE_RES_DSTOVERRUN	0x0200	/* destination overrun */
 538#define	HIFN_BASE_RES_SRCLEN_M		0xc000	/* 17:16 of source count */
 539#define	HIFN_BASE_RES_SRCLEN_S		14
 540#define	HIFN_BASE_RES_DSTLEN_M		0x3000	/* 17:16 of dest count */
 541#define	HIFN_BASE_RES_DSTLEN_S		12
 542
 543struct hifn_comp_result {
 544	volatile __le16		flags;
 545	volatile __le16		crc;
 546};
 547
 548#define	HIFN_COMP_RES_LCB_M		0xff00	/* longitudinal check byte */
 549#define	HIFN_COMP_RES_LCB_S		8
 550#define	HIFN_COMP_RES_RESTART		0x0004	/* MPPC: restart */
 551#define	HIFN_COMP_RES_ENDMARKER		0x0002	/* LZS: end marker seen */
 552#define	HIFN_COMP_RES_SRC_NOTZERO	0x0001	/* source expired */
 553
 554struct hifn_mac_result {
 555	volatile __le16		flags;
 556	volatile __le16		reserved;
 557	/* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
 558};
 559
 560#define	HIFN_MAC_RES_MISCOMPARE		0x0002	/* compare failed */
 561#define	HIFN_MAC_RES_SRC_NOTZERO	0x0001	/* source expired */
 562
 563struct hifn_crypt_result {
 564	volatile __le16		flags;
 565	volatile __le16		reserved;
 566};
 567
 568#define	HIFN_CRYPT_RES_SRC_NOTZERO	0x0001	/* source expired */
 569
 570#ifndef HIFN_POLL_FREQUENCY
 571#define	HIFN_POLL_FREQUENCY	0x1
 572#endif
 573
 574#ifndef HIFN_POLL_SCALAR
 575#define	HIFN_POLL_SCALAR	0x0
 576#endif
 577
 578#define	HIFN_MAX_SEGLEN		0xffff		/* maximum dma segment len */
 579#define	HIFN_MAX_DMALEN		0x3ffff		/* maximum dma length */
 580
 581struct hifn_crypto_alg {
 582	struct list_head	entry;
 583	struct skcipher_alg	alg;
 584	struct hifn_device	*dev;
 585};
 586
 587#define ASYNC_SCATTERLIST_CACHE	16
 588
 589#define ASYNC_FLAGS_MISALIGNED	(1 << 0)
 590
 591struct hifn_cipher_walk {
 592	struct scatterlist	cache[ASYNC_SCATTERLIST_CACHE];
 593	u32			flags;
 594	int			num;
 595};
 596
 597struct hifn_context {
 598	u8			key[HIFN_MAX_CRYPT_KEY_LENGTH];
 599	struct hifn_device	*dev;
 600	unsigned int		keysize;
 601};
 602
 603struct hifn_request_context {
 604	u8			*iv;
 605	unsigned int		ivsize;
 606	u8			op, type, mode, unused;
 607	struct hifn_cipher_walk	walk;
 608};
 609
 610#define crypto_alg_to_hifn(a)	container_of(a, struct hifn_crypto_alg, alg)
 611
 612static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
 613{
 614	return readl(dev->bar[0] + reg);
 615}
 616
 617static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
 618{
 619	return readl(dev->bar[1] + reg);
 620}
 621
 622static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
 623{
 624	writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
 625}
 626
 627static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
 628{
 629	writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
 630}
 631
 632static void hifn_wait_puc(struct hifn_device *dev)
 633{
 634	int i;
 635	u32 ret;
 636
 637	for (i = 10000; i > 0; --i) {
 638		ret = hifn_read_0(dev, HIFN_0_PUCTRL);
 639		if (!(ret & HIFN_PUCTRL_RESET))
 640			break;
 641
 642		udelay(1);
 643	}
 644
 645	if (!i)
 646		dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n");
 647}
 648
 649static void hifn_reset_puc(struct hifn_device *dev)
 650{
 651	hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 652	hifn_wait_puc(dev);
 653}
 654
 655static void hifn_stop_device(struct hifn_device *dev)
 656{
 657	hifn_write_1(dev, HIFN_1_DMA_CSR,
 658		HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 659		HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
 660	hifn_write_0(dev, HIFN_0_PUIER, 0);
 661	hifn_write_1(dev, HIFN_1_DMA_IER, 0);
 662}
 663
 664static void hifn_reset_dma(struct hifn_device *dev, int full)
 665{
 666	hifn_stop_device(dev);
 667
 668	/*
 669	 * Setting poll frequency and others to 0.
 670	 */
 671	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 672			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 673	mdelay(1);
 674
 675	/*
 676	 * Reset DMA.
 677	 */
 678	if (full) {
 679		hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 680		mdelay(1);
 681	} else {
 682		hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
 683				HIFN_DMACNFG_MSTRESET);
 684		hifn_reset_puc(dev);
 685	}
 686
 687	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 688			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 689
 690	hifn_reset_puc(dev);
 691}
 692
 693static u32 hifn_next_signature(u32 a, u_int cnt)
 694{
 695	int i;
 696	u32 v;
 697
 698	for (i = 0; i < cnt; i++) {
 699		/* get the parity */
 700		v = a & 0x80080125;
 701		v ^= v >> 16;
 702		v ^= v >> 8;
 703		v ^= v >> 4;
 704		v ^= v >> 2;
 705		v ^= v >> 1;
 706
 707		a = (v & 1) ^ (a << 1);
 708	}
 709
 710	return a;
 711}
 712
 713static struct pci2id {
 714	u_short		pci_vendor;
 715	u_short		pci_prod;
 716	char		card_id[13];
 717} pci2id[] = {
 718	{
 719		PCI_VENDOR_ID_HIFN,
 720		PCI_DEVICE_ID_HIFN_7955,
 721		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 722		  0x00, 0x00, 0x00, 0x00, 0x00 }
 723	},
 724	{
 725		PCI_VENDOR_ID_HIFN,
 726		PCI_DEVICE_ID_HIFN_7956,
 727		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 728		  0x00, 0x00, 0x00, 0x00, 0x00 }
 729	}
 730};
 731
 732#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 733static int hifn_rng_data_present(struct hwrng *rng, int wait)
 734{
 735	struct hifn_device *dev = (struct hifn_device *)rng->priv;
 736	s64 nsec;
 737
 738	nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
 739	nsec -= dev->rng_wait_time;
 740	if (nsec <= 0)
 741		return 1;
 742	if (!wait)
 743		return 0;
 744	ndelay(nsec);
 745	return 1;
 746}
 747
 748static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
 749{
 750	struct hifn_device *dev = (struct hifn_device *)rng->priv;
 751
 752	*data = hifn_read_1(dev, HIFN_1_RNG_DATA);
 753	dev->rngtime = ktime_get();
 754	return 4;
 755}
 756
 757static int hifn_register_rng(struct hifn_device *dev)
 758{
 759	/*
 760	 * We must wait at least 256 Pk_clk cycles between two reads of the rng.
 761	 */
 762	dev->rng_wait_time	= DIV_ROUND_UP_ULL(NSEC_PER_SEC,
 763						   dev->pk_clk_freq) * 256;
 764
 765	dev->rng.name		= dev->name;
 766	dev->rng.data_present	= hifn_rng_data_present;
 767	dev->rng.data_read	= hifn_rng_data_read;
 768	dev->rng.priv		= (unsigned long)dev;
 769
 770	return hwrng_register(&dev->rng);
 771}
 772
 773static void hifn_unregister_rng(struct hifn_device *dev)
 774{
 775	hwrng_unregister(&dev->rng);
 776}
 777#else
 778#define hifn_register_rng(dev)		0
 779#define hifn_unregister_rng(dev)
 780#endif
 781
 782static int hifn_init_pubrng(struct hifn_device *dev)
 783{
 784	int i;
 785
 786	hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
 787			HIFN_PUBRST_RESET);
 788
 789	for (i = 100; i > 0; --i) {
 790		mdelay(1);
 791
 792		if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
 793			break;
 794	}
 795
 796	if (!i) {
 797		dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n");
 798	} else {
 799		hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 800		dev->dmareg |= HIFN_DMAIER_PUBDONE;
 801		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
 802
 803		dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n");
 804	}
 805
 806	/* Enable RNG engine. */
 807
 808	hifn_write_1(dev, HIFN_1_RNG_CONFIG,
 809			hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
 810	dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n");
 811
 812#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
 813	/* First value must be discarded */
 814	hifn_read_1(dev, HIFN_1_RNG_DATA);
 815	dev->rngtime = ktime_get();
 816#endif
 817	return 0;
 818}
 819
 820static int hifn_enable_crypto(struct hifn_device *dev)
 821{
 822	u32 dmacfg, addr;
 823	char *offtbl = NULL;
 824	int i;
 825
 826	for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
 827		if (pci2id[i].pci_vendor == dev->pdev->vendor &&
 828				pci2id[i].pci_prod == dev->pdev->device) {
 829			offtbl = pci2id[i].card_id;
 830			break;
 831		}
 832	}
 833
 834	if (!offtbl) {
 835		dev_err(&dev->pdev->dev, "Unknown card!\n");
 836		return -ENODEV;
 837	}
 838
 839	dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
 840
 841	hifn_write_1(dev, HIFN_1_DMA_CNFG,
 842			HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
 843			HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 844	mdelay(1);
 845	addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
 846	mdelay(1);
 847	hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
 848	mdelay(1);
 849
 850	for (i = 0; i < 12; ++i) {
 851		addr = hifn_next_signature(addr, offtbl[i] + 0x101);
 852		hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
 853
 854		mdelay(1);
 855	}
 856	hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
 857
 858	dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev));
 859
 860	return 0;
 861}
 862
 863static void hifn_init_dma(struct hifn_device *dev)
 864{
 865	struct hifn_dma *dma = dev->desc_virt;
 866	u32 dptr = dev->desc_dma;
 867	int i;
 868
 869	for (i = 0; i < HIFN_D_CMD_RSIZE; ++i)
 870		dma->cmdr[i].p = __cpu_to_le32(dptr +
 871				offsetof(struct hifn_dma, command_bufs[i][0]));
 872	for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
 873		dma->resr[i].p = __cpu_to_le32(dptr +
 874				offsetof(struct hifn_dma, result_bufs[i][0]));
 875
 876	/* Setup LAST descriptors. */
 877	dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
 878			offsetof(struct hifn_dma, cmdr[0]));
 879	dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
 880			offsetof(struct hifn_dma, srcr[0]));
 881	dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
 882			offsetof(struct hifn_dma, dstr[0]));
 883	dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
 884			offsetof(struct hifn_dma, resr[0]));
 885
 886	dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
 887	dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
 888	dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
 889}
 890
 891/*
 892 * Initialize the PLL. We need to know the frequency of the reference clock
 893 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
 894 * allows us to operate without the risk of overclocking the chip. If it
 895 * actually uses 33MHz, the chip will operate at half the speed, this can be
 896 * overridden by specifying the frequency as module parameter (pci33).
 897 *
 898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a
 899 * stable clock and the PCI clock frequency may vary, so the default is the
 900 * external clock. There is no way to find out its frequency, we default to
 901 * 66MHz since according to Mike Ham of HiFn, almost every board in existence
 902 * has an external crystal populated at 66MHz.
 903 */
 904static void hifn_init_pll(struct hifn_device *dev)
 905{
 906	unsigned int freq, m;
 907	u32 pllcfg;
 908
 909	pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
 910
 911	if (strncmp(hifn_pll_ref, "ext", 3) == 0)
 912		pllcfg |= HIFN_PLL_REF_CLK_PLL;
 913	else
 914		pllcfg |= HIFN_PLL_REF_CLK_HBI;
 915
 916	if (hifn_pll_ref[3] != '\0')
 917		freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
 918	else {
 919		freq = 66;
 920		dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n",
 921			 freq, hifn_pll_ref);
 922	}
 923
 924	m = HIFN_PLL_FCK_MAX / freq;
 925
 926	pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
 927	if (m <= 8)
 928		pllcfg |= HIFN_PLL_IS_1_8;
 929	else
 930		pllcfg |= HIFN_PLL_IS_9_12;
 931
 932	/* Select clock source and enable clock bypass */
 933	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 934		     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
 935
 936	/* Let the chip lock to the input clock */
 937	mdelay(10);
 938
 939	/* Disable clock bypass */
 940	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 941		     HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
 942
 943	/* Switch the engines to the PLL */
 944	hifn_write_1(dev, HIFN_1_PLL, pllcfg |
 945		     HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
 946
 947	/*
 948	 * The Fpk_clk runs at half the total speed. Its frequency is needed to
 949	 * calculate the minimum time between two reads of the rng. Since 33MHz
 950	 * is actually 33.333... we overestimate the frequency here, resulting
 951	 * in slightly larger intervals.
 952	 */
 953	dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
 954}
 955
 956static void hifn_init_registers(struct hifn_device *dev)
 957{
 958	u32 dptr = dev->desc_dma;
 959
 960	/* Initialization magic... */
 961	hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 962	hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
 963	hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
 964
 965	/* write all 4 ring address registers */
 966	hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
 967				offsetof(struct hifn_dma, cmdr[0]));
 968	hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
 969				offsetof(struct hifn_dma, srcr[0]));
 970	hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
 971				offsetof(struct hifn_dma, dstr[0]));
 972	hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
 973				offsetof(struct hifn_dma, resr[0]));
 974
 975	mdelay(2);
 976#if 0
 977	hifn_write_1(dev, HIFN_1_DMA_CSR,
 978	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 979	    HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
 980	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 981	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 982	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 983	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 984	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 985	    HIFN_DMACSR_S_WAIT |
 986	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
 987	    HIFN_DMACSR_C_WAIT |
 988	    HIFN_DMACSR_ENGINE |
 989	    HIFN_DMACSR_PUBDONE);
 990#else
 991	hifn_write_1(dev, HIFN_1_DMA_CSR,
 992	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 993	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
 994	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 995	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 996	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 997	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 998	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 999	    HIFN_DMACSR_S_WAIT |
1000	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1001	    HIFN_DMACSR_C_WAIT |
1002	    HIFN_DMACSR_ENGINE |
1003	    HIFN_DMACSR_PUBDONE);
1004#endif
1005	hifn_read_1(dev, HIFN_1_DMA_CSR);
1006
1007	dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1008	    HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1009	    HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1010	    HIFN_DMAIER_ENGINE;
1011	dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
1012
1013	hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1014	hifn_read_1(dev, HIFN_1_DMA_IER);
1015#if 0
1016	hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
1017		    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1018		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1019		    HIFN_PUCNFG_DRAM);
1020#else
1021	hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
1022#endif
1023	hifn_init_pll(dev);
1024
1025	hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1026	hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1027	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1028	    ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1029	    ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1030}
1031
1032static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
1033		unsigned dlen, unsigned slen, u16 mask, u8 snum)
1034{
1035	struct hifn_base_command *base_cmd;
1036	u8 *buf_pos = buf;
1037
1038	base_cmd = (struct hifn_base_command *)buf_pos;
1039	base_cmd->masks = __cpu_to_le16(mask);
1040	base_cmd->total_source_count =
1041		__cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
1042	base_cmd->total_dest_count =
1043		__cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1044
1045	dlen >>= 16;
1046	slen >>= 16;
1047	base_cmd->session_num = __cpu_to_le16(snum |
1048	    ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1049	    ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1050
1051	return sizeof(struct hifn_base_command);
1052}
1053
1054static int hifn_setup_crypto_command(struct hifn_device *dev,
1055		u8 *buf, unsigned dlen, unsigned slen,
1056		u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
1057{
1058	struct hifn_dma *dma = dev->desc_virt;
1059	struct hifn_crypt_command *cry_cmd;
1060	u8 *buf_pos = buf;
1061	u16 cmd_len;
1062
1063	cry_cmd = (struct hifn_crypt_command *)buf_pos;
1064
1065	cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
1066	dlen >>= 16;
1067	cry_cmd->masks = __cpu_to_le16(mode |
1068			((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
1069			 HIFN_CRYPT_CMD_SRCLEN_M));
1070	cry_cmd->header_skip = 0;
1071	cry_cmd->reserved = 0;
1072
1073	buf_pos += sizeof(struct hifn_crypt_command);
1074
1075	dma->cmdu++;
1076	if (dma->cmdu > 1) {
1077		dev->dmareg |= HIFN_DMAIER_C_WAIT;
1078		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1079	}
1080
1081	if (keylen) {
1082		memcpy(buf_pos, key, keylen);
1083		buf_pos += keylen;
1084	}
1085	if (ivsize) {
1086		memcpy(buf_pos, iv, ivsize);
1087		buf_pos += ivsize;
1088	}
1089
1090	cmd_len = buf_pos - buf;
1091
1092	return cmd_len;
1093}
1094
1095static int hifn_setup_cmd_desc(struct hifn_device *dev,
1096		struct hifn_context *ctx, struct hifn_request_context *rctx,
1097		void *priv, unsigned int nbytes)
1098{
1099	struct hifn_dma *dma = dev->desc_virt;
1100	int cmd_len, sa_idx;
1101	u8 *buf, *buf_pos;
1102	u16 mask;
1103
1104	sa_idx = dma->cmdi;
1105	buf_pos = buf = dma->command_bufs[dma->cmdi];
1106
1107	mask = 0;
1108	switch (rctx->op) {
1109	case ACRYPTO_OP_DECRYPT:
1110		mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1111		break;
1112	case ACRYPTO_OP_ENCRYPT:
1113		mask = HIFN_BASE_CMD_CRYPT;
1114		break;
1115	case ACRYPTO_OP_HMAC:
1116		mask = HIFN_BASE_CMD_MAC;
1117		break;
1118	default:
1119		goto err_out;
1120	}
1121
1122	buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1123			nbytes, mask, dev->snum);
1124
1125	if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
1126		u16 md = 0;
1127
1128		if (ctx->keysize)
1129			md |= HIFN_CRYPT_CMD_NEW_KEY;
1130		if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
1131			md |= HIFN_CRYPT_CMD_NEW_IV;
1132
1133		switch (rctx->mode) {
1134		case ACRYPTO_MODE_ECB:
1135			md |= HIFN_CRYPT_CMD_MODE_ECB;
1136			break;
1137		case ACRYPTO_MODE_CBC:
1138			md |= HIFN_CRYPT_CMD_MODE_CBC;
1139			break;
1140		case ACRYPTO_MODE_CFB:
1141			md |= HIFN_CRYPT_CMD_MODE_CFB;
1142			break;
1143		case ACRYPTO_MODE_OFB:
1144			md |= HIFN_CRYPT_CMD_MODE_OFB;
1145			break;
1146		default:
1147			goto err_out;
1148		}
1149
1150		switch (rctx->type) {
1151		case ACRYPTO_TYPE_AES_128:
1152			if (ctx->keysize != 16)
1153				goto err_out;
1154			md |= HIFN_CRYPT_CMD_KSZ_128 |
1155				HIFN_CRYPT_CMD_ALG_AES;
1156			break;
1157		case ACRYPTO_TYPE_AES_192:
1158			if (ctx->keysize != 24)
1159				goto err_out;
1160			md |= HIFN_CRYPT_CMD_KSZ_192 |
1161				HIFN_CRYPT_CMD_ALG_AES;
1162			break;
1163		case ACRYPTO_TYPE_AES_256:
1164			if (ctx->keysize != 32)
1165				goto err_out;
1166			md |= HIFN_CRYPT_CMD_KSZ_256 |
1167				HIFN_CRYPT_CMD_ALG_AES;
1168			break;
1169		case ACRYPTO_TYPE_3DES:
1170			if (ctx->keysize != 24)
1171				goto err_out;
1172			md |= HIFN_CRYPT_CMD_ALG_3DES;
1173			break;
1174		case ACRYPTO_TYPE_DES:
1175			if (ctx->keysize != 8)
1176				goto err_out;
1177			md |= HIFN_CRYPT_CMD_ALG_DES;
1178			break;
1179		default:
1180			goto err_out;
1181		}
1182
1183		buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1184				nbytes, nbytes, ctx->key, ctx->keysize,
1185				rctx->iv, rctx->ivsize, md);
1186	}
1187
1188	dev->sa[sa_idx] = priv;
1189	dev->started++;
1190
1191	cmd_len = buf_pos - buf;
1192	dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1193			HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1194
1195	if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1196		dma->cmdr[dma->cmdi].l = __cpu_to_le32(
1197			HIFN_D_VALID | HIFN_D_LAST |
1198			HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1199		dma->cmdi = 0;
1200	} else {
1201		dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID);
1202	}
1203
1204	if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1205		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1206		dev->flags |= HIFN_FLAG_CMD_BUSY;
1207	}
1208	return 0;
1209
1210err_out:
1211	return -EINVAL;
1212}
1213
1214static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1215		unsigned int offset, unsigned int size, int last)
1216{
1217	struct hifn_dma *dma = dev->desc_virt;
1218	int idx;
1219	dma_addr_t addr;
1220
1221	addr = dma_map_page(&dev->pdev->dev, page, offset, size,
1222			    DMA_TO_DEVICE);
1223
1224	idx = dma->srci;
1225
1226	dma->srcr[idx].p = __cpu_to_le32(addr);
1227	dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1228			HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1229
1230	if (++idx == HIFN_D_SRC_RSIZE) {
1231		dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1232				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1233				(last ? HIFN_D_LAST : 0));
1234		idx = 0;
1235	}
1236
1237	dma->srci = idx;
1238	dma->srcu++;
1239
1240	if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1241		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1242		dev->flags |= HIFN_FLAG_SRC_BUSY;
1243	}
1244
1245	return size;
1246}
1247
1248static void hifn_setup_res_desc(struct hifn_device *dev)
1249{
1250	struct hifn_dma *dma = dev->desc_virt;
1251
1252	dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1253			HIFN_D_VALID | HIFN_D_LAST);
1254	/*
1255	 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1256	 *					HIFN_D_LAST);
1257	 */
1258
1259	if (++dma->resi == HIFN_D_RES_RSIZE) {
1260		dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1261				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1262		dma->resi = 0;
1263	}
1264
1265	dma->resu++;
1266
1267	if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1268		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1269		dev->flags |= HIFN_FLAG_RES_BUSY;
1270	}
1271}
1272
1273static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1274		unsigned offset, unsigned size, int last)
1275{
1276	struct hifn_dma *dma = dev->desc_virt;
1277	int idx;
1278	dma_addr_t addr;
1279
1280	addr = dma_map_page(&dev->pdev->dev, page, offset, size,
1281			    DMA_FROM_DEVICE);
1282
1283	idx = dma->dsti;
1284	dma->dstr[idx].p = __cpu_to_le32(addr);
1285	dma->dstr[idx].l = __cpu_to_le32(size |	HIFN_D_VALID |
1286			HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
1287
1288	if (++idx == HIFN_D_DST_RSIZE) {
1289		dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1290				HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1291				(last ? HIFN_D_LAST : 0));
1292		idx = 0;
1293	}
1294	dma->dsti = idx;
1295	dma->dstu++;
1296
1297	if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1298		hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1299		dev->flags |= HIFN_FLAG_DST_BUSY;
1300	}
1301}
1302
1303static int hifn_setup_dma(struct hifn_device *dev,
1304		struct hifn_context *ctx, struct hifn_request_context *rctx,
1305		struct scatterlist *src, struct scatterlist *dst,
1306		unsigned int nbytes, void *priv)
1307{
1308	struct scatterlist *t;
1309	struct page *spage, *dpage;
1310	unsigned int soff, doff;
1311	unsigned int n, len;
1312
1313	n = nbytes;
1314	while (n) {
1315		spage = sg_page(src);
1316		soff = src->offset;
1317		len = min(src->length, n);
1318
1319		hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
1320
1321		src++;
1322		n -= len;
1323	}
1324
1325	t = &rctx->walk.cache[0];
1326	n = nbytes;
1327	while (n) {
1328		if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1329			BUG_ON(!sg_page(t));
1330			dpage = sg_page(t);
1331			doff = 0;
1332			len = t->length;
1333		} else {
1334			BUG_ON(!sg_page(dst));
1335			dpage = sg_page(dst);
1336			doff = dst->offset;
1337			len = dst->length;
1338		}
1339		len = min(len, n);
1340
1341		hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
1342
1343		dst++;
1344		t++;
1345		n -= len;
1346	}
1347
1348	hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
1349	hifn_setup_res_desc(dev);
1350	return 0;
1351}
1352
1353static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
1354		int num, gfp_t gfp_flags)
1355{
1356	int i;
1357
1358	num = min(ASYNC_SCATTERLIST_CACHE, num);
1359	sg_init_table(w->cache, num);
1360
1361	w->num = 0;
1362	for (i = 0; i < num; ++i) {
1363		struct page *page = alloc_page(gfp_flags);
1364		struct scatterlist *s;
1365
1366		if (!page)
1367			break;
1368
1369		s = &w->cache[i];
1370
1371		sg_set_page(s, page, PAGE_SIZE, 0);
1372		w->num++;
1373	}
1374
1375	return i;
1376}
1377
1378static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
1379{
1380	int i;
1381
1382	for (i = 0; i < w->num; ++i) {
1383		struct scatterlist *s = &w->cache[i];
1384
1385		__free_page(sg_page(s));
1386
1387		s->length = 0;
1388	}
1389
1390	w->num = 0;
1391}
1392
1393static int skcipher_add(unsigned int *drestp, struct scatterlist *dst,
1394		unsigned int size, unsigned int *nbytesp)
1395{
1396	unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1397	int idx = 0;
1398
1399	if (drest < size || size > nbytes)
1400		return -EINVAL;
1401
1402	while (size) {
1403		copy = min3(drest, size, dst->length);
1404
1405		size -= copy;
1406		drest -= copy;
1407		nbytes -= copy;
1408
1409		pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1410			 __func__, copy, size, drest, nbytes);
1411
1412		dst++;
1413		idx++;
1414	}
1415
1416	*nbytesp = nbytes;
1417	*drestp = drest;
1418
1419	return idx;
1420}
1421
1422static int hifn_cipher_walk(struct skcipher_request *req,
1423		struct hifn_cipher_walk *w)
1424{
1425	struct scatterlist *dst, *t;
1426	unsigned int nbytes = req->cryptlen, offset, copy, diff;
1427	int idx, tidx, err;
1428
1429	tidx = idx = 0;
1430	offset = 0;
1431	while (nbytes) {
1432		if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1433			return -EINVAL;
1434
1435		dst = &req->dst[idx];
1436
1437		pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
1438			 __func__, dst->length, dst->offset, offset, nbytes);
1439
1440		if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1441		    !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
1442		    offset) {
1443			unsigned slen = min(dst->length - offset, nbytes);
1444			unsigned dlen = PAGE_SIZE;
1445
1446			t = &w->cache[idx];
1447
1448			err = skcipher_add(&dlen, dst, slen, &nbytes);
1449			if (err < 0)
1450				return err;
1451
1452			idx += err;
1453
1454			copy = slen & ~(HIFN_D_DST_DALIGN - 1);
1455			diff = slen & (HIFN_D_DST_DALIGN - 1);
1456
1457			if (dlen < nbytes) {
1458				/*
1459				 * Destination page does not have enough space
1460				 * to put there additional blocksized chunk,
1461				 * so we mark that page as containing only
1462				 * blocksize aligned chunks:
1463				 *	t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
1464				 * and increase number of bytes to be processed
1465				 * in next chunk:
1466				 *	nbytes += diff;
1467				 */
1468				nbytes += diff;
1469
1470				/*
1471				 * Temporary of course...
1472				 * Kick author if you will catch this one.
1473				 */
1474				pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n",
1475				       __func__, dlen, nbytes, slen, offset);
1476				pr_err("%s: please contact author to fix this "
1477				       "issue, generally you should not catch "
1478				       "this path under any condition but who "
1479				       "knows how did you use crypto code.\n"
1480				       "Thank you.\n",	__func__);
1481				BUG();
1482			} else {
1483				copy += diff + nbytes;
1484
1485				dst = &req->dst[idx];
1486
1487				err = skcipher_add(&dlen, dst, nbytes, &nbytes);
1488				if (err < 0)
1489					return err;
1490
1491				idx += err;
1492			}
1493
1494			t->length = copy;
1495			t->offset = offset;
1496		} else {
1497			nbytes -= min(dst->length, nbytes);
1498			idx++;
1499		}
1500
1501		tidx++;
1502	}
1503
1504	return tidx;
1505}
1506
1507static int hifn_setup_session(struct skcipher_request *req)
1508{
1509	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1510	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1511	struct hifn_device *dev = ctx->dev;
1512	unsigned long dlen, flags;
1513	unsigned int nbytes = req->cryptlen, idx = 0;
1514	int err = -EINVAL, sg_num;
1515	struct scatterlist *dst;
1516
1517	if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
1518		goto err_out_exit;
1519
1520	rctx->walk.flags = 0;
1521
1522	while (nbytes) {
1523		dst = &req->dst[idx];
1524		dlen = min(dst->length, nbytes);
1525
1526		if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1527		    !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
1528			rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1529
1530		nbytes -= dlen;
1531		idx++;
1532	}
1533
1534	if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1535		err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
1536		if (err < 0)
1537			return err;
1538	}
1539
1540	sg_num = hifn_cipher_walk(req, &rctx->walk);
1541	if (sg_num < 0) {
1542		err = sg_num;
1543		goto err_out_exit;
1544	}
1545
1546	spin_lock_irqsave(&dev->lock, flags);
1547	if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1548		err = -EAGAIN;
1549		goto err_out;
1550	}
1551
1552	err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req);
1553	if (err)
1554		goto err_out;
1555
1556	dev->snum++;
1557
1558	dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1559	spin_unlock_irqrestore(&dev->lock, flags);
1560
1561	return 0;
1562
1563err_out:
1564	spin_unlock_irqrestore(&dev->lock, flags);
1565err_out_exit:
1566	if (err) {
1567		dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1568			 "type: %u, err: %d.\n",
1569			 rctx->iv, rctx->ivsize,
1570			 ctx->key, ctx->keysize,
1571			 rctx->mode, rctx->op, rctx->type, err);
1572	}
1573
1574	return err;
1575}
1576
1577static int hifn_start_device(struct hifn_device *dev)
1578{
1579	int err;
1580
1581	dev->started = dev->active = 0;
1582	hifn_reset_dma(dev, 1);
1583
1584	err = hifn_enable_crypto(dev);
1585	if (err)
1586		return err;
1587
1588	hifn_reset_puc(dev);
1589
1590	hifn_init_dma(dev);
1591
1592	hifn_init_registers(dev);
1593
1594	hifn_init_pubrng(dev);
1595
1596	return 0;
1597}
1598
1599static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1600		struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1601{
1602	unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1603	void *daddr;
1604	int idx = 0;
1605
1606	if (srest < size || size > nbytes)
1607		return -EINVAL;
1608
1609	while (size) {
1610		copy = min3(srest, dst->length, size);
1611
1612		daddr = kmap_atomic(sg_page(dst));
1613		memcpy(daddr + dst->offset + offset, saddr, copy);
1614		kunmap_atomic(daddr);
1615
1616		nbytes -= copy;
1617		size -= copy;
1618		srest -= copy;
1619		saddr += copy;
1620		offset = 0;
1621
1622		pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1623			 __func__, copy, size, srest, nbytes);
1624
1625		dst++;
1626		idx++;
1627	}
1628
1629	*nbytesp = nbytes;
1630	*srestp = srest;
1631
1632	return idx;
1633}
1634
1635static inline void hifn_complete_sa(struct hifn_device *dev, int i)
1636{
1637	unsigned long flags;
1638
1639	spin_lock_irqsave(&dev->lock, flags);
1640	dev->sa[i] = NULL;
1641	dev->started--;
1642	if (dev->started < 0)
1643		dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__,
1644			 dev->started);
1645	spin_unlock_irqrestore(&dev->lock, flags);
1646	BUG_ON(dev->started < 0);
1647}
1648
1649static void hifn_process_ready(struct skcipher_request *req, int error)
1650{
1651	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1652
1653	if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1654		unsigned int nbytes = req->cryptlen;
1655		int idx = 0, err;
1656		struct scatterlist *dst, *t;
1657		void *saddr;
1658
1659		while (nbytes) {
1660			t = &rctx->walk.cache[idx];
1661			dst = &req->dst[idx];
1662
1663			pr_debug("\n%s: sg_page(t): %p, t->length: %u, "
1664				"sg_page(dst): %p, dst->length: %u, "
1665				"nbytes: %u.\n",
1666				__func__, sg_page(t), t->length,
1667				sg_page(dst), dst->length, nbytes);
1668
1669			if (!t->length) {
1670				nbytes -= min(dst->length, nbytes);
1671				idx++;
1672				continue;
1673			}
1674
1675			saddr = kmap_atomic(sg_page(t));
1676
1677			err = skcipher_get(saddr, &t->length, t->offset,
1678					dst, nbytes, &nbytes);
1679			if (err < 0) {
1680				kunmap_atomic(saddr);
1681				break;
1682			}
1683
1684			idx += err;
1685			kunmap_atomic(saddr);
1686		}
1687
1688		hifn_cipher_walk_exit(&rctx->walk);
1689	}
1690
1691	skcipher_request_complete(req, error);
1692}
1693
1694static void hifn_clear_rings(struct hifn_device *dev, int error)
1695{
1696	struct hifn_dma *dma = dev->desc_virt;
1697	int i, u;
1698
1699	dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1700			"k: %d.%d.%d.%d.\n",
1701			dma->cmdi, dma->srci, dma->dsti, dma->resi,
1702			dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1703			dma->cmdk, dma->srck, dma->dstk, dma->resk);
1704
1705	i = dma->resk; u = dma->resu;
1706	while (u != 0) {
1707		if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1708			break;
1709
1710		if (dev->sa[i]) {
1711			dev->success++;
1712			dev->reset = 0;
1713			hifn_process_ready(dev->sa[i], error);
1714			hifn_complete_sa(dev, i);
1715		}
1716
1717		if (++i == HIFN_D_RES_RSIZE)
1718			i = 0;
1719		u--;
1720	}
1721	dma->resk = i; dma->resu = u;
1722
1723	i = dma->srck; u = dma->srcu;
1724	while (u != 0) {
1725		if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1726			break;
1727		if (++i == HIFN_D_SRC_RSIZE)
1728			i = 0;
1729		u--;
1730	}
1731	dma->srck = i; dma->srcu = u;
1732
1733	i = dma->cmdk; u = dma->cmdu;
1734	while (u != 0) {
1735		if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1736			break;
1737		if (++i == HIFN_D_CMD_RSIZE)
1738			i = 0;
1739		u--;
1740	}
1741	dma->cmdk = i; dma->cmdu = u;
1742
1743	i = dma->dstk; u = dma->dstu;
1744	while (u != 0) {
1745		if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1746			break;
1747		if (++i == HIFN_D_DST_RSIZE)
1748			i = 0;
1749		u--;
1750	}
1751	dma->dstk = i; dma->dstu = u;
1752
1753	dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1754			"k: %d.%d.%d.%d.\n",
1755			dma->cmdi, dma->srci, dma->dsti, dma->resi,
1756			dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1757			dma->cmdk, dma->srck, dma->dstk, dma->resk);
1758}
1759
1760static void hifn_work(struct work_struct *work)
1761{
1762	struct delayed_work *dw = to_delayed_work(work);
1763	struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1764	unsigned long flags;
1765	int reset = 0;
1766	u32 r = 0;
1767
1768	spin_lock_irqsave(&dev->lock, flags);
1769	if (dev->active == 0) {
1770		struct hifn_dma *dma = dev->desc_virt;
1771
1772		if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1773			dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1774			r |= HIFN_DMACSR_C_CTRL_DIS;
1775		}
1776		if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1777			dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1778			r |= HIFN_DMACSR_S_CTRL_DIS;
1779		}
1780		if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1781			dev->flags &= ~HIFN_FLAG_DST_BUSY;
1782			r |= HIFN_DMACSR_D_CTRL_DIS;
1783		}
1784		if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1785			dev->flags &= ~HIFN_FLAG_RES_BUSY;
1786			r |= HIFN_DMACSR_R_CTRL_DIS;
1787		}
1788		if (r)
1789			hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1790	} else
1791		dev->active--;
1792
1793	if ((dev->prev_success == dev->success) && dev->started)
1794		reset = 1;
1795	dev->prev_success = dev->success;
1796	spin_unlock_irqrestore(&dev->lock, flags);
1797
1798	if (reset) {
1799		if (++dev->reset >= 5) {
1800			int i;
1801			struct hifn_dma *dma = dev->desc_virt;
1802
1803			dev_info(&dev->pdev->dev,
1804				 "r: %08x, active: %d, started: %d, "
1805				 "success: %lu: qlen: %u/%u, reset: %d.\n",
1806				 r, dev->active, dev->started,
1807				 dev->success, dev->queue.qlen, dev->queue.max_qlen,
1808				 reset);
1809
1810			dev_info(&dev->pdev->dev, "%s: res: ", __func__);
1811			for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1812				pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]);
1813				if (dev->sa[i]) {
1814					hifn_process_ready(dev->sa[i], -ENODEV);
1815					hifn_complete_sa(dev, i);
1816				}
1817			}
1818			pr_info("\n");
1819
1820			hifn_reset_dma(dev, 1);
1821			hifn_stop_device(dev);
1822			hifn_start_device(dev);
1823			dev->reset = 0;
1824		}
1825
1826		tasklet_schedule(&dev->tasklet);
1827	}
1828
1829	schedule_delayed_work(&dev->work, HZ);
1830}
1831
1832static irqreturn_t hifn_interrupt(int irq, void *data)
1833{
1834	struct hifn_device *dev = data;
1835	struct hifn_dma *dma = dev->desc_virt;
1836	u32 dmacsr, restart;
1837
1838	dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
1839
1840	dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
1841			"i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
1842		dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
1843		dma->cmdi, dma->srci, dma->dsti, dma->resi,
1844		dma->cmdu, dma->srcu, dma->dstu, dma->resu);
1845
1846	if ((dmacsr & dev->dmareg) == 0)
1847		return IRQ_NONE;
1848
1849	hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
1850
1851	if (dmacsr & HIFN_DMACSR_ENGINE)
1852		hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
1853	if (dmacsr & HIFN_DMACSR_PUBDONE)
1854		hifn_write_1(dev, HIFN_1_PUB_STATUS,
1855			hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
1856
1857	restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
1858	if (restart) {
1859		u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
1860
1861		dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
1862			 !!(dmacsr & HIFN_DMACSR_R_OVER),
1863			 !!(dmacsr & HIFN_DMACSR_D_OVER),
1864			puisr, !!(puisr & HIFN_PUISR_DSTOVER));
1865		if (!!(puisr & HIFN_PUISR_DSTOVER))
1866			hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1867		hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
1868					HIFN_DMACSR_D_OVER));
1869	}
1870
1871	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
1872			HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
1873	if (restart) {
1874		dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n",
1875			 !!(dmacsr & HIFN_DMACSR_C_ABORT),
1876			 !!(dmacsr & HIFN_DMACSR_S_ABORT),
1877			 !!(dmacsr & HIFN_DMACSR_D_ABORT),
1878			 !!(dmacsr & HIFN_DMACSR_R_ABORT));
1879		hifn_reset_dma(dev, 1);
1880		hifn_init_dma(dev);
1881		hifn_init_registers(dev);
1882	}
1883
1884	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
1885		dev_dbg(&dev->pdev->dev, "wait on command.\n");
1886		dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
1887		hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1888	}
1889
1890	tasklet_schedule(&dev->tasklet);
1891
1892	return IRQ_HANDLED;
1893}
1894
1895static void hifn_flush(struct hifn_device *dev)
1896{
1897	unsigned long flags;
1898	struct crypto_async_request *async_req;
1899	struct skcipher_request *req;
1900	struct hifn_dma *dma = dev->desc_virt;
1901	int i;
1902
1903	for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
1904		struct hifn_desc *d = &dma->resr[i];
1905
1906		if (dev->sa[i]) {
1907			hifn_process_ready(dev->sa[i],
1908				(d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0);
1909			hifn_complete_sa(dev, i);
1910		}
1911	}
1912
1913	spin_lock_irqsave(&dev->lock, flags);
1914	while ((async_req = crypto_dequeue_request(&dev->queue))) {
1915		req = skcipher_request_cast(async_req);
1916		spin_unlock_irqrestore(&dev->lock, flags);
1917
1918		hifn_process_ready(req, -ENODEV);
1919
1920		spin_lock_irqsave(&dev->lock, flags);
1921	}
1922	spin_unlock_irqrestore(&dev->lock, flags);
1923}
1924
1925static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key,
1926		unsigned int len)
1927{
1928	struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
1929	struct hifn_device *dev = ctx->dev;
1930	int err;
1931
1932	err = verify_skcipher_des_key(cipher, key);
1933	if (err)
1934		return err;
1935
1936	dev->flags &= ~HIFN_FLAG_OLD_KEY;
1937
1938	memcpy(ctx->key, key, len);
1939	ctx->keysize = len;
1940
1941	return 0;
1942}
1943
1944static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
1945			    unsigned int len)
1946{
1947	struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
1948	struct hifn_device *dev = ctx->dev;
1949	int err;
1950
1951	err = verify_skcipher_des3_key(cipher, key);
1952	if (err)
1953		return err;
1954
1955	dev->flags &= ~HIFN_FLAG_OLD_KEY;
1956
1957	memcpy(ctx->key, key, len);
1958	ctx->keysize = len;
1959
1960	return 0;
1961}
1962
1963static int hifn_handle_req(struct skcipher_request *req)
1964{
1965	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1966	struct hifn_device *dev = ctx->dev;
1967	int err = -EAGAIN;
1968
1969	if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
1970		err = hifn_setup_session(req);
1971
1972	if (err == -EAGAIN) {
1973		unsigned long flags;
1974
1975		spin_lock_irqsave(&dev->lock, flags);
1976		err = crypto_enqueue_request(&dev->queue, &req->base);
1977		spin_unlock_irqrestore(&dev->lock, flags);
1978	}
1979
1980	return err;
1981}
1982
1983static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op,
1984		u8 type, u8 mode)
1985{
1986	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1987	struct hifn_request_context *rctx = skcipher_request_ctx(req);
1988	unsigned ivsize;
1989
1990	ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1991
1992	if (req->iv && mode != ACRYPTO_MODE_ECB) {
1993		if (type == ACRYPTO_TYPE_AES_128)
1994			ivsize = HIFN_AES_IV_LENGTH;
1995		else if (type == ACRYPTO_TYPE_DES)
1996			ivsize = HIFN_DES_KEY_LENGTH;
1997		else if (type == ACRYPTO_TYPE_3DES)
1998			ivsize = HIFN_3DES_KEY_LENGTH;
1999	}
2000
2001	if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
2002		if (ctx->keysize == 24)
2003			type = ACRYPTO_TYPE_AES_192;
2004		else if (ctx->keysize == 32)
2005			type = ACRYPTO_TYPE_AES_256;
2006	}
2007
2008	rctx->op = op;
2009	rctx->mode = mode;
2010	rctx->type = type;
2011	rctx->iv = req->iv;
2012	rctx->ivsize = ivsize;
2013
2014	/*
2015	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2016	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2017	 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2018	 */
2019
2020	return hifn_handle_req(req);
2021}
2022
2023static int hifn_process_queue(struct hifn_device *dev)
2024{
2025	struct crypto_async_request *async_req, *backlog;
2026	struct skcipher_request *req;
2027	unsigned long flags;
2028	int err = 0;
2029
2030	while (dev->started < HIFN_QUEUE_LENGTH) {
2031		spin_lock_irqsave(&dev->lock, flags);
2032		backlog = crypto_get_backlog(&dev->queue);
2033		async_req = crypto_dequeue_request(&dev->queue);
2034		spin_unlock_irqrestore(&dev->lock, flags);
2035
2036		if (!async_req)
2037			break;
2038
2039		if (backlog)
2040			crypto_request_complete(backlog, -EINPROGRESS);
2041
2042		req = skcipher_request_cast(async_req);
2043
2044		err = hifn_handle_req(req);
2045		if (err)
2046			break;
2047	}
2048
2049	return err;
2050}
2051
2052static int hifn_setup_crypto(struct skcipher_request *req, u8 op,
2053		u8 type, u8 mode)
2054{
2055	int err;
2056	struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2057	struct hifn_device *dev = ctx->dev;
2058
2059	err = hifn_setup_crypto_req(req, op, type, mode);
2060	if (err)
2061		return err;
2062
2063	if (dev->started < HIFN_QUEUE_LENGTH &&	dev->queue.qlen)
2064		hifn_process_queue(dev);
2065
2066	return -EINPROGRESS;
2067}
2068
2069/*
2070 * AES ecryption functions.
2071 */
2072static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req)
2073{
2074	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2075			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2076}
2077static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req)
2078{
2079	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2080			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2081}
 
 
 
 
 
 
 
 
 
 
2082
2083/*
2084 * AES decryption functions.
2085 */
2086static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req)
2087{
2088	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2089			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2090}
2091static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req)
2092{
2093	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2094			ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2095}
 
 
 
 
 
 
 
 
 
 
2096
2097/*
2098 * DES ecryption functions.
2099 */
2100static inline int hifn_encrypt_des_ecb(struct skcipher_request *req)
2101{
2102	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2103			ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2104}
2105static inline int hifn_encrypt_des_cbc(struct skcipher_request *req)
2106{
2107	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2108			ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2109}
 
 
 
 
 
 
 
 
 
 
2110
2111/*
2112 * DES decryption functions.
2113 */
2114static inline int hifn_decrypt_des_ecb(struct skcipher_request *req)
2115{
2116	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2117			ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2118}
2119static inline int hifn_decrypt_des_cbc(struct skcipher_request *req)
2120{
2121	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2122			ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2123}
 
 
 
 
 
 
 
 
 
 
2124
2125/*
2126 * 3DES ecryption functions.
2127 */
2128static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req)
2129{
2130	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2131			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2132}
2133static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req)
2134{
2135	return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2136			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2137}
 
 
 
 
 
 
 
 
 
 
2138
2139/* 3DES decryption functions. */
2140static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req)
2141{
2142	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2143			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2144}
2145static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req)
2146{
2147	return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2148			ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2149}
 
 
 
 
 
 
 
 
 
 
2150
2151struct hifn_alg_template {
2152	char name[CRYPTO_MAX_ALG_NAME];
2153	char drv_name[CRYPTO_MAX_ALG_NAME];
2154	unsigned int bsize;
2155	struct skcipher_alg skcipher;
2156};
2157
2158static const struct hifn_alg_template hifn_alg_templates[] = {
2159	/*
2160	 * 3DES ECB and CBC modes.
2161	 */
2162	{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2163		.name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2164		.skcipher = {
2165			.ivsize		=	HIFN_IV_LENGTH,
2166			.min_keysize	=	HIFN_3DES_KEY_LENGTH,
2167			.max_keysize	=	HIFN_3DES_KEY_LENGTH,
2168			.setkey		=	hifn_des3_setkey,
2169			.encrypt	=	hifn_encrypt_3des_cbc,
2170			.decrypt	=	hifn_decrypt_3des_cbc,
2171		},
2172	},
2173	{
2174		.name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2175		.skcipher = {
2176			.min_keysize	=	HIFN_3DES_KEY_LENGTH,
2177			.max_keysize	=	HIFN_3DES_KEY_LENGTH,
2178			.setkey		=	hifn_des3_setkey,
2179			.encrypt	=	hifn_encrypt_3des_ecb,
2180			.decrypt	=	hifn_decrypt_3des_ecb,
2181		},
2182	},
2183
2184	/*
2185	 * DES ECB and CBC modes.
2186	 */
2187	{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2188		.name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2189		.skcipher = {
2190			.ivsize		=	HIFN_IV_LENGTH,
2191			.min_keysize	=	HIFN_DES_KEY_LENGTH,
2192			.max_keysize	=	HIFN_DES_KEY_LENGTH,
2193			.setkey		=	hifn_setkey,
2194			.encrypt	=	hifn_encrypt_des_cbc,
2195			.decrypt	=	hifn_decrypt_des_cbc,
2196		},
2197	},
2198	{
2199		.name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2200		.skcipher = {
2201			.min_keysize	=	HIFN_DES_KEY_LENGTH,
2202			.max_keysize	=	HIFN_DES_KEY_LENGTH,
2203			.setkey		=	hifn_setkey,
2204			.encrypt	=	hifn_encrypt_des_ecb,
2205			.decrypt	=	hifn_decrypt_des_ecb,
2206		},
2207	},
2208
2209	/*
2210	 * AES ECB and CBC modes.
2211	 */
2212	{
2213		.name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2214		.skcipher = {
2215			.min_keysize	=	AES_MIN_KEY_SIZE,
2216			.max_keysize	=	AES_MAX_KEY_SIZE,
2217			.setkey		=	hifn_setkey,
2218			.encrypt	=	hifn_encrypt_aes_ecb,
2219			.decrypt	=	hifn_decrypt_aes_ecb,
2220		},
2221	},
2222	{
2223		.name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2224		.skcipher = {
2225			.ivsize		=	HIFN_AES_IV_LENGTH,
2226			.min_keysize	=	AES_MIN_KEY_SIZE,
2227			.max_keysize	=	AES_MAX_KEY_SIZE,
2228			.setkey		=	hifn_setkey,
2229			.encrypt	=	hifn_encrypt_aes_cbc,
2230			.decrypt	=	hifn_decrypt_aes_cbc,
2231		},
2232	},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2233};
2234
2235static int hifn_init_tfm(struct crypto_skcipher *tfm)
2236{
2237	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2238	struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2239	struct hifn_context *ctx = crypto_skcipher_ctx(tfm);
2240
2241	ctx->dev = ha->dev;
2242	crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context));
2243
2244	return 0;
2245}
2246
2247static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t)
2248{
2249	struct hifn_crypto_alg *alg;
2250	int err;
2251
2252	alg = kzalloc(sizeof(*alg), GFP_KERNEL);
2253	if (!alg)
2254		return -ENOMEM;
2255
2256	alg->alg = t->skcipher;
2257	alg->alg.init = hifn_init_tfm;
2258
2259	err = -EINVAL;
2260	if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
2261		     "%s", t->name) >= CRYPTO_MAX_ALG_NAME)
2262		goto out_free_alg;
2263	if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
2264		     "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME)
2265		goto out_free_alg;
2266
2267	alg->alg.base.cra_priority = 300;
2268	alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
2269	alg->alg.base.cra_blocksize = t->bsize;
2270	alg->alg.base.cra_ctxsize = sizeof(struct hifn_context);
2271	alg->alg.base.cra_alignmask = 0;
2272	alg->alg.base.cra_module = THIS_MODULE;
2273
2274	alg->dev = dev;
2275
2276	list_add_tail(&alg->entry, &dev->alg_list);
2277
2278	err = crypto_register_skcipher(&alg->alg);
2279	if (err) {
2280		list_del(&alg->entry);
2281out_free_alg:
2282		kfree(alg);
2283	}
2284
2285	return err;
2286}
2287
2288static void hifn_unregister_alg(struct hifn_device *dev)
2289{
2290	struct hifn_crypto_alg *a, *n;
2291
2292	list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2293		list_del(&a->entry);
2294		crypto_unregister_skcipher(&a->alg);
2295		kfree(a);
2296	}
2297}
2298
2299static int hifn_register_alg(struct hifn_device *dev)
2300{
2301	int i, err;
2302
2303	for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) {
2304		err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2305		if (err)
2306			goto err_out_exit;
2307	}
2308
2309	return 0;
2310
2311err_out_exit:
2312	hifn_unregister_alg(dev);
2313	return err;
2314}
2315
2316static void hifn_tasklet_callback(unsigned long data)
2317{
2318	struct hifn_device *dev = (struct hifn_device *)data;
2319
2320	/*
2321	 * This is ok to call this without lock being held,
2322	 * althogh it modifies some parameters used in parallel,
2323	 * (like dev->success), but they are used in process
2324	 * context or update is atomic (like setting dev->sa[i] to NULL).
2325	 */
2326	hifn_clear_rings(dev, 0);
2327
2328	if (dev->started < HIFN_QUEUE_LENGTH &&	dev->queue.qlen)
2329		hifn_process_queue(dev);
2330}
2331
2332static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2333{
2334	int err, i;
2335	struct hifn_device *dev;
2336	char name[8];
2337
2338	err = pci_enable_device(pdev);
2339	if (err)
2340		return err;
2341	pci_set_master(pdev);
2342
2343	err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2344	if (err)
2345		goto err_out_disable_pci_device;
2346
2347	snprintf(name, sizeof(name), "hifn%d",
2348			atomic_inc_return(&hifn_dev_number) - 1);
2349
2350	err = pci_request_regions(pdev, name);
2351	if (err)
2352		goto err_out_disable_pci_device;
2353
2354	if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2355	    pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2356	    pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2357		dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n");
2358		err = -ENODEV;
2359		goto err_out_free_regions;
2360	}
2361
2362	dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2363			GFP_KERNEL);
2364	if (!dev) {
2365		err = -ENOMEM;
2366		goto err_out_free_regions;
2367	}
2368
2369	INIT_LIST_HEAD(&dev->alg_list);
2370
2371	snprintf(dev->name, sizeof(dev->name), "%s", name);
2372	spin_lock_init(&dev->lock);
2373
2374	for (i = 0; i < 3; ++i) {
2375		unsigned long addr, size;
2376
2377		addr = pci_resource_start(pdev, i);
2378		size = pci_resource_len(pdev, i);
2379
2380		dev->bar[i] = ioremap(addr, size);
2381		if (!dev->bar[i]) {
2382			err = -ENOMEM;
2383			goto err_out_unmap_bars;
2384		}
2385	}
2386
2387	dev->desc_virt = dma_alloc_coherent(&pdev->dev,
2388					    sizeof(struct hifn_dma),
2389					    &dev->desc_dma, GFP_KERNEL);
2390	if (!dev->desc_virt) {
2391		dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");
2392		err = -ENOMEM;
2393		goto err_out_unmap_bars;
2394	}
2395
2396	dev->pdev = pdev;
2397	dev->irq = pdev->irq;
2398
2399	for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
2400		dev->sa[i] = NULL;
2401
2402	pci_set_drvdata(pdev, dev);
2403
2404	tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
2405
2406	crypto_init_queue(&dev->queue, 1);
2407
2408	err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2409	if (err) {
2410		dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n",
2411			dev->irq, err);
2412		dev->irq = 0;
2413		goto err_out_free_desc;
2414	}
2415
2416	err = hifn_start_device(dev);
2417	if (err)
2418		goto err_out_free_irq;
2419
2420	err = hifn_register_rng(dev);
2421	if (err)
2422		goto err_out_stop_device;
2423
2424	err = hifn_register_alg(dev);
2425	if (err)
2426		goto err_out_unregister_rng;
2427
2428	INIT_DELAYED_WORK(&dev->work, hifn_work);
2429	schedule_delayed_work(&dev->work, HZ);
2430
2431	dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been "
2432		"successfully registered as %s.\n",
2433		pci_name(pdev), dev->name);
2434
2435	return 0;
2436
2437err_out_unregister_rng:
2438	hifn_unregister_rng(dev);
2439err_out_stop_device:
2440	hifn_reset_dma(dev, 1);
2441	hifn_stop_device(dev);
2442err_out_free_irq:
2443	free_irq(dev->irq, dev);
2444	tasklet_kill(&dev->tasklet);
2445err_out_free_desc:
2446	dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt,
2447			  dev->desc_dma);
2448
2449err_out_unmap_bars:
2450	for (i = 0; i < 3; ++i)
2451		if (dev->bar[i])
2452			iounmap(dev->bar[i]);
2453	kfree(dev);
2454
2455err_out_free_regions:
2456	pci_release_regions(pdev);
2457
2458err_out_disable_pci_device:
2459	pci_disable_device(pdev);
2460
2461	return err;
2462}
2463
2464static void hifn_remove(struct pci_dev *pdev)
2465{
2466	int i;
2467	struct hifn_device *dev;
2468
2469	dev = pci_get_drvdata(pdev);
2470
2471	if (dev) {
2472		cancel_delayed_work_sync(&dev->work);
2473
2474		hifn_unregister_rng(dev);
2475		hifn_unregister_alg(dev);
2476		hifn_reset_dma(dev, 1);
2477		hifn_stop_device(dev);
2478
2479		free_irq(dev->irq, dev);
2480		tasklet_kill(&dev->tasklet);
2481
2482		hifn_flush(dev);
2483
2484		dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma),
2485				  dev->desc_virt, dev->desc_dma);
2486		for (i = 0; i < 3; ++i)
2487			if (dev->bar[i])
2488				iounmap(dev->bar[i]);
2489
2490		kfree(dev);
2491	}
2492
2493	pci_release_regions(pdev);
2494	pci_disable_device(pdev);
2495}
2496
2497static struct pci_device_id hifn_pci_tbl[] = {
2498	{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2499	{ PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2500	{ 0 }
2501};
2502MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2503
2504static struct pci_driver hifn_pci_driver = {
2505	.name     = "hifn795x",
2506	.id_table = hifn_pci_tbl,
2507	.probe    = hifn_probe,
2508	.remove   = hifn_remove,
2509};
2510
2511static int __init hifn_init(void)
2512{
2513	unsigned int freq;
2514	int err;
 
 
 
2515
2516	if (strncmp(hifn_pll_ref, "ext", 3) &&
2517	    strncmp(hifn_pll_ref, "pci", 3)) {
2518		pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");
2519		return -EINVAL;
2520	}
2521
2522	/*
2523	 * For the 7955/7956 the reference clock frequency must be in the
2524	 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
2525	 * but this chip is currently not supported.
2526	 */
2527	if (hifn_pll_ref[3] != '\0') {
2528		freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
2529		if (freq < 20 || freq > 100) {
2530			pr_err("hifn795x: invalid hifn_pll_ref frequency, must"
2531			       "be in the range of 20-100");
2532			return -EINVAL;
2533		}
2534	}
2535
2536	err = pci_register_driver(&hifn_pci_driver);
2537	if (err < 0) {
2538		pr_err("Failed to register PCI driver for %s device.\n",
2539		       hifn_pci_driver.name);
2540		return -ENODEV;
2541	}
2542
2543	pr_info("Driver for HIFN 795x crypto accelerator chip "
2544		"has been successfully registered.\n");
2545
2546	return 0;
2547}
2548
2549static void __exit hifn_fini(void)
2550{
2551	pci_unregister_driver(&hifn_pci_driver);
2552
2553	pr_info("Driver for HIFN 795x crypto accelerator chip "
2554		"has been successfully unregistered.\n");
2555}
2556
2557module_init(hifn_init);
2558module_exit(hifn_fini);
2559
2560MODULE_LICENSE("GPL");
2561MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2562MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");