1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
   4 *
   5 * Derived from:
   6 *	https://github.com/yuq/sunxi-nfc-mtd
   7 *	Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
   8 *
   9 *	https://github.com/hno/Allwinner-Info
  10 *	Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
  11 *
  12 *	Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
  13 *	Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
 
 
 
 
 
 
 
 
 
 
  14 */
  15
  16#include <linux/dma-mapping.h>
  17#include <linux/slab.h>
  18#include <linux/module.h>
  19#include <linux/moduleparam.h>
  20#include <linux/platform_device.h>
  21#include <linux/of.h>
 
 
  22#include <linux/mtd/mtd.h>
  23#include <linux/mtd/rawnand.h>
  24#include <linux/mtd/partitions.h>
  25#include <linux/clk.h>
  26#include <linux/delay.h>
  27#include <linux/dmaengine.h>
 
  28#include <linux/interrupt.h>
  29#include <linux/iopoll.h>
  30#include <linux/reset.h>
  31
  32#define NFC_REG_CTL		0x0000
  33#define NFC_REG_ST		0x0004
  34#define NFC_REG_INT		0x0008
  35#define NFC_REG_TIMING_CTL	0x000C
  36#define NFC_REG_TIMING_CFG	0x0010
  37#define NFC_REG_ADDR_LOW	0x0014
  38#define NFC_REG_ADDR_HIGH	0x0018
  39#define NFC_REG_SECTOR_NUM	0x001C
  40#define NFC_REG_CNT		0x0020
  41#define NFC_REG_CMD		0x0024
  42#define NFC_REG_RCMD_SET	0x0028
  43#define NFC_REG_WCMD_SET	0x002C
  44#define NFC_REG_A10_IO_DATA	0x0030
  45#define NFC_REG_A23_IO_DATA	0x0300
  46#define NFC_REG_ECC_CTL		0x0034
  47#define NFC_REG_ECC_ST		0x0038
  48#define NFC_REG_DEBUG		0x003C
  49#define NFC_REG_ECC_ERR_CNT(x)	((0x0040 + (x)) & ~0x3)
  50#define NFC_REG_USER_DATA(x)	(0x0050 + ((x) * 4))
  51#define NFC_REG_SPARE_AREA	0x00A0
  52#define NFC_REG_PAT_ID		0x00A4
  53#define NFC_REG_MDMA_ADDR	0x00C0
  54#define NFC_REG_MDMA_CNT	0x00C4
  55#define NFC_RAM0_BASE		0x0400
  56#define NFC_RAM1_BASE		0x0800
  57
  58/* define bit use in NFC_CTL */
  59#define NFC_EN			BIT(0)
  60#define NFC_RESET		BIT(1)
  61#define NFC_BUS_WIDTH_MSK	BIT(2)
  62#define NFC_BUS_WIDTH_8		(0 << 2)
  63#define NFC_BUS_WIDTH_16	(1 << 2)
  64#define NFC_RB_SEL_MSK		BIT(3)
  65#define NFC_RB_SEL(x)		((x) << 3)
  66#define NFC_CE_SEL_MSK		GENMASK(26, 24)
  67#define NFC_CE_SEL(x)		((x) << 24)
  68#define NFC_CE_CTL		BIT(6)
  69#define NFC_PAGE_SHIFT_MSK	GENMASK(11, 8)
  70#define NFC_PAGE_SHIFT(x)	(((x) < 10 ? 0 : (x) - 10) << 8)
  71#define NFC_SAM			BIT(12)
  72#define NFC_RAM_METHOD		BIT(14)
  73#define NFC_DMA_TYPE_NORMAL	BIT(15)
  74#define NFC_DEBUG_CTL		BIT(31)
  75
  76/* define bit use in NFC_ST */
  77#define NFC_RB_B2R		BIT(0)
  78#define NFC_CMD_INT_FLAG	BIT(1)
  79#define NFC_DMA_INT_FLAG	BIT(2)
  80#define NFC_CMD_FIFO_STATUS	BIT(3)
  81#define NFC_STA			BIT(4)
  82#define NFC_NATCH_INT_FLAG	BIT(5)
  83#define NFC_RB_STATE(x)		BIT(x + 8)
  84
  85/* define bit use in NFC_INT */
  86#define NFC_B2R_INT_ENABLE	BIT(0)
  87#define NFC_CMD_INT_ENABLE	BIT(1)
  88#define NFC_DMA_INT_ENABLE	BIT(2)
  89#define NFC_INT_MASK		(NFC_B2R_INT_ENABLE | \
  90				 NFC_CMD_INT_ENABLE | \
  91				 NFC_DMA_INT_ENABLE)
  92
  93/* define bit use in NFC_TIMING_CTL */
  94#define NFC_TIMING_CTL_EDO	BIT(8)
  95
  96/* define NFC_TIMING_CFG register layout */
  97#define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD)		\
  98	(((tWB) & 0x3) | (((tADL) & 0x3) << 2) |		\
  99	(((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) |		\
 100	(((tCAD) & 0x7) << 8))
 101
 102/* define bit use in NFC_CMD */
 103#define NFC_CMD_LOW_BYTE_MSK	GENMASK(7, 0)
 104#define NFC_CMD_HIGH_BYTE_MSK	GENMASK(15, 8)
 105#define NFC_CMD(x)		(x)
 106#define NFC_ADR_NUM_MSK		GENMASK(18, 16)
 107#define NFC_ADR_NUM(x)		(((x) - 1) << 16)
 108#define NFC_SEND_ADR		BIT(19)
 109#define NFC_ACCESS_DIR		BIT(20)
 110#define NFC_DATA_TRANS		BIT(21)
 111#define NFC_SEND_CMD1		BIT(22)
 112#define NFC_WAIT_FLAG		BIT(23)
 113#define NFC_SEND_CMD2		BIT(24)
 114#define NFC_SEQ			BIT(25)
 115#define NFC_DATA_SWAP_METHOD	BIT(26)
 116#define NFC_ROW_AUTO_INC	BIT(27)
 117#define NFC_SEND_CMD3		BIT(28)
 118#define NFC_SEND_CMD4		BIT(29)
 119#define NFC_CMD_TYPE_MSK	GENMASK(31, 30)
 120#define NFC_NORMAL_OP		(0 << 30)
 121#define NFC_ECC_OP		(1 << 30)
 122#define NFC_PAGE_OP		(2U << 30)
 123
 124/* define bit use in NFC_RCMD_SET */
 125#define NFC_READ_CMD_MSK	GENMASK(7, 0)
 126#define NFC_RND_READ_CMD0_MSK	GENMASK(15, 8)
 127#define NFC_RND_READ_CMD1_MSK	GENMASK(23, 16)
 128
 129/* define bit use in NFC_WCMD_SET */
 130#define NFC_PROGRAM_CMD_MSK	GENMASK(7, 0)
 131#define NFC_RND_WRITE_CMD_MSK	GENMASK(15, 8)
 132#define NFC_READ_CMD0_MSK	GENMASK(23, 16)
 133#define NFC_READ_CMD1_MSK	GENMASK(31, 24)
 134
 135/* define bit use in NFC_ECC_CTL */
 136#define NFC_ECC_EN		BIT(0)
 137#define NFC_ECC_PIPELINE	BIT(3)
 138#define NFC_ECC_EXCEPTION	BIT(4)
 139#define NFC_ECC_BLOCK_SIZE_MSK	BIT(5)
 140#define NFC_ECC_BLOCK_512	BIT(5)
 141#define NFC_RANDOM_EN		BIT(9)
 142#define NFC_RANDOM_DIRECTION	BIT(10)
 143#define NFC_ECC_MODE_MSK	GENMASK(15, 12)
 144#define NFC_ECC_MODE(x)		((x) << 12)
 145#define NFC_RANDOM_SEED_MSK	GENMASK(30, 16)
 146#define NFC_RANDOM_SEED(x)	((x) << 16)
 147
 148/* define bit use in NFC_ECC_ST */
 149#define NFC_ECC_ERR(x)		BIT(x)
 150#define NFC_ECC_ERR_MSK		GENMASK(15, 0)
 151#define NFC_ECC_PAT_FOUND(x)	BIT(x + 16)
 152#define NFC_ECC_ERR_CNT(b, x)	(((x) >> (((b) % 4) * 8)) & 0xff)
 153
 154#define NFC_DEFAULT_TIMEOUT_MS	1000
 155
 156#define NFC_SRAM_SIZE		1024
 157
 158#define NFC_MAX_CS		7
 159
 160/**
 161 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 162 *
 163 * @cs: the NAND CS id used to communicate with a NAND Chip
 164 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
 165 */
 166struct sunxi_nand_chip_sel {
 167	u8 cs;
 168	s8 rb;
 169};
 170
 171/**
 172 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
 173 *
 174 * @ecc_ctl: ECC_CTL register value for this NAND chip
 175 */
 176struct sunxi_nand_hw_ecc {
 177	u32 ecc_ctl;
 178};
 179
 180/**
 181 * struct sunxi_nand_chip - stores NAND chip device related information
 182 *
 183 * @node: used to store NAND chips into a list
 184 * @nand: base NAND chip structure
 185 * @ecc: ECC controller structure
 186 * @clk_rate: clk_rate required for this NAND chip
 187 * @timing_cfg: TIMING_CFG register value for this NAND chip
 188 * @timing_ctl: TIMING_CTL register value for this NAND chip
 189 * @nsels: number of CS lines required by the NAND chip
 190 * @sels: array of CS lines descriptions
 191 */
 192struct sunxi_nand_chip {
 193	struct list_head node;
 194	struct nand_chip nand;
 195	struct sunxi_nand_hw_ecc ecc;
 196	unsigned long clk_rate;
 197	u32 timing_cfg;
 198	u32 timing_ctl;
 
 
 
 
 
 199	int nsels;
 200	struct sunxi_nand_chip_sel sels[] __counted_by(nsels);
 201};
 202
 203static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
 204{
 205	return container_of(nand, struct sunxi_nand_chip, nand);
 206}
 207
 208/*
 209 * NAND Controller capabilities structure: stores NAND controller capabilities
 210 * for distinction between compatible strings.
 211 *
 212 * @has_mdma:		Use mbus dma mode, otherwise general dma
 213 *			through MBUS on A23/A33 needs extra configuration.
 214 * @reg_io_data:	I/O data register
 215 * @dma_maxburst:	DMA maxburst
 216 */
 217struct sunxi_nfc_caps {
 218	bool has_mdma;
 219	unsigned int reg_io_data;
 220	unsigned int dma_maxburst;
 221};
 222
 223/**
 224 * struct sunxi_nfc - stores sunxi NAND controller information
 225 *
 226 * @controller: base controller structure
 227 * @dev: parent device (used to print error messages)
 228 * @regs: NAND controller registers
 229 * @ahb_clk: NAND controller AHB clock
 230 * @mod_clk: NAND controller mod clock
 231 * @reset: NAND controller reset line
 232 * @assigned_cs: bitmask describing already assigned CS lines
 233 * @clk_rate: NAND controller current clock rate
 234 * @chips: a list containing all the NAND chips attached to this NAND
 235 *	   controller
 236 * @complete: a completion object used to wait for NAND controller events
 237 * @dmac: the DMA channel attached to the NAND controller
 238 * @caps: NAND Controller capabilities
 239 */
 240struct sunxi_nfc {
 241	struct nand_controller controller;
 242	struct device *dev;
 243	void __iomem *regs;
 244	struct clk *ahb_clk;
 245	struct clk *mod_clk;
 246	struct reset_control *reset;
 247	unsigned long assigned_cs;
 248	unsigned long clk_rate;
 249	struct list_head chips;
 250	struct completion complete;
 251	struct dma_chan *dmac;
 252	const struct sunxi_nfc_caps *caps;
 253};
 254
 255static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
 256{
 257	return container_of(ctrl, struct sunxi_nfc, controller);
 258}
 259
 260static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
 261{
 262	struct sunxi_nfc *nfc = dev_id;
 263	u32 st = readl(nfc->regs + NFC_REG_ST);
 264	u32 ien = readl(nfc->regs + NFC_REG_INT);
 265
 266	if (!(ien & st))
 267		return IRQ_NONE;
 268
 269	if ((ien & st) == ien)
 270		complete(&nfc->complete);
 271
 272	writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 273	writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
 274
 275	return IRQ_HANDLED;
 276}
 277
 278static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
 279				 bool use_polling, unsigned int timeout_ms)
 280{
 281	int ret;
 282
 283	if (events & ~NFC_INT_MASK)
 284		return -EINVAL;
 285
 286	if (!timeout_ms)
 287		timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
 288
 289	if (!use_polling) {
 290		init_completion(&nfc->complete);
 291
 292		writel(events, nfc->regs + NFC_REG_INT);
 293
 294		ret = wait_for_completion_timeout(&nfc->complete,
 295						msecs_to_jiffies(timeout_ms));
 296		if (!ret)
 297			ret = -ETIMEDOUT;
 298		else
 299			ret = 0;
 300
 301		writel(0, nfc->regs + NFC_REG_INT);
 302	} else {
 303		u32 status;
 304
 305		ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 306					 (status & events) == events, 1,
 307					 timeout_ms * 1000);
 308	}
 309
 310	writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
 311
 312	if (ret)
 313		dev_err(nfc->dev, "wait interrupt timedout\n");
 314
 315	return ret;
 316}
 317
 318static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
 319{
 320	u32 status;
 321	int ret;
 322
 323	ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
 324				 !(status & NFC_CMD_FIFO_STATUS), 1,
 325				 NFC_DEFAULT_TIMEOUT_MS * 1000);
 326	if (ret)
 327		dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
 328
 329	return ret;
 330}
 331
 332static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
 333{
 334	u32 ctl;
 335	int ret;
 336
 337	writel(0, nfc->regs + NFC_REG_ECC_CTL);
 338	writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
 339
 340	ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
 341				 !(ctl & NFC_RESET), 1,
 342				 NFC_DEFAULT_TIMEOUT_MS * 1000);
 343	if (ret)
 344		dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
 345
 346	return ret;
 347}
 348
 349static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
 350				    int chunksize, int nchunks,
 351				    enum dma_data_direction ddir,
 352				    struct scatterlist *sg)
 353{
 
 
 354	struct dma_async_tx_descriptor *dmad;
 355	enum dma_transfer_direction tdir;
 356	dma_cookie_t dmat;
 357	int ret;
 358
 359	if (ddir == DMA_FROM_DEVICE)
 360		tdir = DMA_DEV_TO_MEM;
 361	else
 362		tdir = DMA_MEM_TO_DEV;
 363
 364	sg_init_one(sg, buf, nchunks * chunksize);
 365	ret = dma_map_sg(nfc->dev, sg, 1, ddir);
 366	if (!ret)
 367		return -ENOMEM;
 368
 369	if (!nfc->caps->has_mdma) {
 370		dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
 371		if (!dmad) {
 372			ret = -EINVAL;
 373			goto err_unmap_buf;
 374		}
 375	}
 376
 377	writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
 378	       nfc->regs + NFC_REG_CTL);
 379	writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
 380	writel(chunksize, nfc->regs + NFC_REG_CNT);
 
 381
 382	if (nfc->caps->has_mdma) {
 383		writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,
 384		       nfc->regs + NFC_REG_CTL);
 385		writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT);
 386		writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR);
 387	} else {
 388		dmat = dmaengine_submit(dmad);
 389
 390		ret = dma_submit_error(dmat);
 391		if (ret)
 392			goto err_clr_dma_flag;
 393	}
 394
 395	return 0;
 396
 397err_clr_dma_flag:
 398	writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 399	       nfc->regs + NFC_REG_CTL);
 400
 401err_unmap_buf:
 402	dma_unmap_sg(nfc->dev, sg, 1, ddir);
 403	return ret;
 404}
 405
 406static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
 407				     enum dma_data_direction ddir,
 408				     struct scatterlist *sg)
 409{
 
 
 
 410	dma_unmap_sg(nfc->dev, sg, 1, ddir);
 411	writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
 412	       nfc->regs + NFC_REG_CTL);
 413}
 414
 415static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 416{
 417	struct mtd_info *mtd = nand_to_mtd(nand);
 418	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 419	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 420	struct sunxi_nand_chip_sel *sel;
 421	u32 ctl;
 422
 423	if (cs >= sunxi_nand->nsels)
 
 
 
 424		return;
 425
 426	ctl = readl(nfc->regs + NFC_REG_CTL) &
 427	      ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
 428
 429	sel = &sunxi_nand->sels[cs];
 430	ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
 431	if (sel->rb >= 0)
 432		ctl |= NFC_RB_SEL(sel->rb);
 433
 434	writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
 435
 436	if (nfc->clk_rate != sunxi_nand->clk_rate) {
 437		clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
 438		nfc->clk_rate = sunxi_nand->clk_rate;
 
 
 
 
 
 
 
 
 
 439	}
 440
 441	writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
 442	writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
 443	writel(ctl, nfc->regs + NFC_REG_CTL);
 
 
 444}
 445
 446static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
 447{
 
 448	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 449	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 450	int ret;
 451	int cnt;
 452	int offs = 0;
 453	u32 tmp;
 454
 455	while (len > offs) {
 456		bool poll = false;
 457
 458		cnt = min(len - offs, NFC_SRAM_SIZE);
 459
 460		ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 461		if (ret)
 462			break;
 463
 464		writel(cnt, nfc->regs + NFC_REG_CNT);
 465		tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
 466		writel(tmp, nfc->regs + NFC_REG_CMD);
 467
 468		/* Arbitrary limit for polling mode */
 469		if (cnt < 64)
 470			poll = true;
 471
 472		ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 473		if (ret)
 474			break;
 475
 476		if (buf)
 477			memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
 478				      cnt);
 479		offs += cnt;
 480	}
 481}
 482
 483static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
 484				int len)
 485{
 
 486	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 487	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 488	int ret;
 489	int cnt;
 490	int offs = 0;
 491	u32 tmp;
 492
 493	while (len > offs) {
 494		bool poll = false;
 495
 496		cnt = min(len - offs, NFC_SRAM_SIZE);
 497
 498		ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 499		if (ret)
 500			break;
 501
 502		writel(cnt, nfc->regs + NFC_REG_CNT);
 503		memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
 504		tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
 505		      NFC_ACCESS_DIR;
 506		writel(tmp, nfc->regs + NFC_REG_CMD);
 507
 508		/* Arbitrary limit for polling mode */
 509		if (cnt < 64)
 510			poll = true;
 511
 512		ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
 513		if (ret)
 514			break;
 515
 516		offs += cnt;
 517	}
 518}
 519
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 520/* These seed values have been extracted from Allwinner's BSP */
 521static const u16 sunxi_nfc_randomizer_page_seeds[] = {
 522	0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
 523	0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
 524	0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
 525	0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
 526	0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
 527	0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
 528	0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
 529	0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
 530	0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
 531	0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
 532	0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
 533	0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
 534	0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
 535	0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
 536	0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
 537	0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
 538};
 539
 540/*
 541 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
 542 * have been generated using
 543 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
 544 * the randomizer engine does internally before de/scrambling OOB data.
 545 *
 546 * Those tables are statically defined to avoid calculating randomizer state
 547 * at runtime.
 548 */
 549static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
 550	0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
 551	0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
 552	0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
 553	0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
 554	0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
 555	0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
 556	0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
 557	0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
 558	0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
 559	0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
 560	0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
 561	0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
 562	0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
 563	0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
 564	0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
 565	0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
 566};
 567
 568static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
 569	0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
 570	0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
 571	0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
 572	0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
 573	0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
 574	0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
 575	0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
 576	0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
 577	0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
 578	0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
 579	0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
 580	0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
 581	0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
 582	0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
 583	0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
 584	0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
 585};
 586
 587static u16 sunxi_nfc_randomizer_step(u16 state, int count)
 588{
 589	state &= 0x7fff;
 590
 591	/*
 592	 * This loop is just a simple implementation of a Fibonacci LFSR using
 593	 * the x16 + x15 + 1 polynomial.
 594	 */
 595	while (count--)
 596		state = ((state >> 1) |
 597			 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
 598
 599	return state;
 600}
 601
 602static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
 603				      bool ecc)
 604{
 605	struct mtd_info *mtd = nand_to_mtd(nand);
 606	const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
 607	int mod = mtd_div_by_ws(mtd->erasesize, mtd);
 608
 609	if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
 610		mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
 611
 612	if (ecc) {
 613		if (mtd->ecc_step_size == 512)
 614			seeds = sunxi_nfc_randomizer_ecc512_seeds;
 615		else
 616			seeds = sunxi_nfc_randomizer_ecc1024_seeds;
 617	}
 618
 619	return seeds[page % mod];
 620}
 621
 622static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
 623					bool ecc)
 624{
 
 625	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 626	u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 627	u16 state;
 628
 629	if (!(nand->options & NAND_NEED_SCRAMBLING))
 630		return;
 631
 632	ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
 633	state = sunxi_nfc_randomizer_state(nand, page, ecc);
 634	ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
 635	writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
 636}
 637
 638static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
 639{
 
 640	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 641
 642	if (!(nand->options & NAND_NEED_SCRAMBLING))
 643		return;
 644
 645	writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
 646	       nfc->regs + NFC_REG_ECC_CTL);
 647}
 648
 649static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
 650{
 
 651	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 652
 653	if (!(nand->options & NAND_NEED_SCRAMBLING))
 654		return;
 655
 656	writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
 657	       nfc->regs + NFC_REG_ECC_CTL);
 658}
 659
 660static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
 661{
 662	u16 state = sunxi_nfc_randomizer_state(nand, page, true);
 663
 664	bbm[0] ^= state;
 665	bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
 666}
 667
 668static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
 669					   const uint8_t *buf, int len,
 670					   bool ecc, int page)
 671{
 672	sunxi_nfc_randomizer_config(nand, page, ecc);
 673	sunxi_nfc_randomizer_enable(nand);
 674	sunxi_nfc_write_buf(nand, buf, len);
 675	sunxi_nfc_randomizer_disable(nand);
 676}
 677
 678static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
 679					  int len, bool ecc, int page)
 680{
 681	sunxi_nfc_randomizer_config(nand, page, ecc);
 682	sunxi_nfc_randomizer_enable(nand);
 683	sunxi_nfc_read_buf(nand, buf, len);
 684	sunxi_nfc_randomizer_disable(nand);
 685}
 686
 687static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
 688{
 689	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 690	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 
 
 
 
 
 
 
 
 691
 692	writel(sunxi_nand->ecc.ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
 
 
 
 693}
 694
 695static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
 696{
 
 697	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 698
 699	writel(0, nfc->regs + NFC_REG_ECC_CTL);
 
 700}
 701
 702static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
 703{
 704	buf[0] = user_data;
 705	buf[1] = user_data >> 8;
 706	buf[2] = user_data >> 16;
 707	buf[3] = user_data >> 24;
 708}
 709
 710static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
 711{
 712	return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
 713}
 714
 715static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
 716						int step, bool bbm, int page)
 717{
 
 718	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 719
 720	sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
 721				   oob);
 722
 723	/* De-randomize the Bad Block Marker. */
 724	if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
 725		sunxi_nfc_randomize_bbm(nand, page, oob);
 726}
 727
 728static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 729						const u8 *oob, int step,
 730						bool bbm, int page)
 731{
 
 732	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 733	u8 user_data[4];
 734
 735	/* Randomize the Bad Block Marker. */
 736	if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
 737		memcpy(user_data, oob, sizeof(user_data));
 738		sunxi_nfc_randomize_bbm(nand, page, user_data);
 739		oob = user_data;
 740	}
 741
 742	writel(sunxi_nfc_buf_to_user_data(oob),
 743	       nfc->regs + NFC_REG_USER_DATA(step));
 744}
 745
 746static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
 747					  unsigned int *max_bitflips, int ret)
 748{
 749	struct mtd_info *mtd = nand_to_mtd(nand);
 750
 751	if (ret < 0) {
 752		mtd->ecc_stats.failed++;
 753	} else {
 754		mtd->ecc_stats.corrected += ret;
 755		*max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 756	}
 757}
 758
 759static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
 760				    int step, u32 status, bool *erased)
 761{
 
 762	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 763	struct nand_ecc_ctrl *ecc = &nand->ecc;
 764	u32 tmp;
 765
 766	*erased = false;
 767
 768	if (status & NFC_ECC_ERR(step))
 769		return -EBADMSG;
 770
 771	if (status & NFC_ECC_PAT_FOUND(step)) {
 772		u8 pattern;
 773
 774		if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
 775			pattern = 0x0;
 776		} else {
 777			pattern = 0xff;
 778			*erased = true;
 779		}
 780
 781		if (data)
 782			memset(data, pattern, ecc->size);
 783
 784		if (oob)
 785			memset(oob, pattern, ecc->bytes + 4);
 786
 787		return 0;
 788	}
 789
 790	tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
 791
 792	return NFC_ECC_ERR_CNT(step, tmp);
 793}
 794
 795static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
 796				       u8 *data, int data_off,
 797				       u8 *oob, int oob_off,
 798				       int *cur_off,
 799				       unsigned int *max_bitflips,
 800				       bool bbm, bool oob_required, int page)
 801{
 
 802	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 803	struct nand_ecc_ctrl *ecc = &nand->ecc;
 804	int raw_mode = 0;
 805	bool erased;
 806	int ret;
 807
 808	if (*cur_off != data_off)
 809		nand_change_read_column_op(nand, data_off, NULL, 0, false);
 810
 811	sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
 812
 813	if (data_off + ecc->size != oob_off)
 814		nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 815
 816	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 817	if (ret)
 818		return ret;
 819
 820	sunxi_nfc_randomizer_enable(nand);
 821	writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
 822	       nfc->regs + NFC_REG_CMD);
 823
 824	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
 825	sunxi_nfc_randomizer_disable(nand);
 826	if (ret)
 827		return ret;
 828
 829	*cur_off = oob_off + ecc->bytes + 4;
 830
 831	ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
 832				       readl(nfc->regs + NFC_REG_ECC_ST),
 833				       &erased);
 834	if (erased)
 835		return 1;
 836
 837	if (ret < 0) {
 838		/*
 839		 * Re-read the data with the randomizer disabled to identify
 840		 * bitflips in erased pages.
 841		 */
 842		if (nand->options & NAND_NEED_SCRAMBLING)
 843			nand_change_read_column_op(nand, data_off, data,
 844						   ecc->size, false);
 845		else
 846			memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 847				      ecc->size);
 848
 849		nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
 850					   false);
 851
 852		ret = nand_check_erased_ecc_chunk(data,	ecc->size,
 853						  oob, ecc->bytes + 4,
 854						  NULL, 0, ecc->strength);
 855		if (ret >= 0)
 856			raw_mode = 1;
 857	} else {
 858		memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 859
 860		if (oob_required) {
 861			nand_change_read_column_op(nand, oob_off, NULL, 0,
 862						   false);
 863			sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
 864						      true, page);
 865
 866			sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
 867							    bbm, page);
 868		}
 869	}
 870
 871	sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
 872
 873	return raw_mode;
 874}
 875
 876static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
 877					    u8 *oob, int *cur_off,
 878					    bool randomize, int page)
 879{
 880	struct mtd_info *mtd = nand_to_mtd(nand);
 881	struct nand_ecc_ctrl *ecc = &nand->ecc;
 882	int offset = ((ecc->bytes + 4) * ecc->steps);
 883	int len = mtd->oobsize - offset;
 884
 885	if (len <= 0)
 886		return;
 887
 888	if (!cur_off || *cur_off != offset)
 889		nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
 890					   false);
 891
 892	if (!randomize)
 893		sunxi_nfc_read_buf(nand, oob + offset, len);
 894	else
 895		sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
 896					      false, page);
 897
 898	if (cur_off)
 899		*cur_off = mtd->oobsize + mtd->writesize;
 900}
 901
 902static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
 903					    int oob_required, int page,
 904					    int nchunks)
 905{
 
 906	bool randomized = nand->options & NAND_NEED_SCRAMBLING;
 907	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 908	struct mtd_info *mtd = nand_to_mtd(nand);
 909	struct nand_ecc_ctrl *ecc = &nand->ecc;
 910	unsigned int max_bitflips = 0;
 911	int ret, i, raw_mode = 0;
 912	struct scatterlist sg;
 913	u32 status, wait;
 914
 915	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 916	if (ret)
 917		return ret;
 918
 919	ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
 920				       DMA_FROM_DEVICE, &sg);
 921	if (ret)
 922		return ret;
 923
 924	sunxi_nfc_hw_ecc_enable(nand);
 925	sunxi_nfc_randomizer_config(nand, page, false);
 926	sunxi_nfc_randomizer_enable(nand);
 927
 928	writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
 929	       NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
 930
 931	wait = NFC_CMD_INT_FLAG;
 932
 933	if (nfc->caps->has_mdma)
 934		wait |= NFC_DMA_INT_FLAG;
 935	else
 936		dma_async_issue_pending(nfc->dmac);
 937
 938	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
 939	       nfc->regs + NFC_REG_CMD);
 940
 941	ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
 942	if (ret && !nfc->caps->has_mdma)
 943		dmaengine_terminate_all(nfc->dmac);
 944
 945	sunxi_nfc_randomizer_disable(nand);
 946	sunxi_nfc_hw_ecc_disable(nand);
 947
 948	sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
 949
 950	if (ret)
 951		return ret;
 952
 953	status = readl(nfc->regs + NFC_REG_ECC_ST);
 954
 955	for (i = 0; i < nchunks; i++) {
 956		int data_off = i * ecc->size;
 957		int oob_off = i * (ecc->bytes + 4);
 958		u8 *data = buf + data_off;
 959		u8 *oob = nand->oob_poi + oob_off;
 960		bool erased;
 961
 962		ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
 963					       oob_required ? oob : NULL,
 964					       i, status, &erased);
 965
 966		/* ECC errors are handled in the second loop. */
 967		if (ret < 0)
 968			continue;
 969
 970		if (oob_required && !erased) {
 971			/* TODO: use DMA to retrieve OOB */
 972			nand_change_read_column_op(nand,
 973						   mtd->writesize + oob_off,
 974						   oob, ecc->bytes + 4, false);
 975
 976			sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
 977							    !i, page);
 978		}
 979
 980		if (erased)
 981			raw_mode = 1;
 982
 983		sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
 984	}
 985
 986	if (status & NFC_ECC_ERR_MSK) {
 987		for (i = 0; i < nchunks; i++) {
 988			int data_off = i * ecc->size;
 989			int oob_off = i * (ecc->bytes + 4);
 990			u8 *data = buf + data_off;
 991			u8 *oob = nand->oob_poi + oob_off;
 992
 993			if (!(status & NFC_ECC_ERR(i)))
 994				continue;
 995
 996			/*
 997			 * Re-read the data with the randomizer disabled to
 998			 * identify bitflips in erased pages.
 999			 * TODO: use DMA to read page in raw mode
1000			 */
1001			if (randomized)
1002				nand_change_read_column_op(nand, data_off,
1003							   data, ecc->size,
1004							   false);
1005
1006			/* TODO: use DMA to retrieve OOB */
1007			nand_change_read_column_op(nand,
1008						   mtd->writesize + oob_off,
1009						   oob, ecc->bytes + 4, false);
1010
1011			ret = nand_check_erased_ecc_chunk(data,	ecc->size,
1012							  oob, ecc->bytes + 4,
1013							  NULL, 0,
1014							  ecc->strength);
1015			if (ret >= 0)
1016				raw_mode = 1;
1017
1018			sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1019		}
1020	}
1021
1022	if (oob_required)
1023		sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1024						NULL, !raw_mode,
1025						page);
1026
1027	return max_bitflips;
1028}
1029
1030static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1031					const u8 *data, int data_off,
1032					const u8 *oob, int oob_off,
1033					int *cur_off, bool bbm,
1034					int page)
1035{
 
1036	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1037	struct nand_ecc_ctrl *ecc = &nand->ecc;
1038	int ret;
1039
1040	if (data_off != *cur_off)
1041		nand_change_write_column_op(nand, data_off, NULL, 0, false);
1042
1043	sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1044
1045	if (data_off + ecc->size != oob_off)
1046		nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1047
1048	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1049	if (ret)
1050		return ret;
1051
1052	sunxi_nfc_randomizer_enable(nand);
1053	sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1054
1055	writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1056	       NFC_ACCESS_DIR | NFC_ECC_OP,
1057	       nfc->regs + NFC_REG_CMD);
1058
1059	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1060	sunxi_nfc_randomizer_disable(nand);
1061	if (ret)
1062		return ret;
1063
1064	*cur_off = oob_off + ecc->bytes + 4;
1065
1066	return 0;
1067}
1068
1069static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1070					     u8 *oob, int *cur_off,
1071					     int page)
1072{
1073	struct mtd_info *mtd = nand_to_mtd(nand);
1074	struct nand_ecc_ctrl *ecc = &nand->ecc;
1075	int offset = ((ecc->bytes + 4) * ecc->steps);
1076	int len = mtd->oobsize - offset;
1077
1078	if (len <= 0)
1079		return;
1080
1081	if (!cur_off || *cur_off != offset)
1082		nand_change_write_column_op(nand, offset + mtd->writesize,
1083					    NULL, 0, false);
1084
1085	sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1086
1087	if (cur_off)
1088		*cur_off = mtd->oobsize + mtd->writesize;
1089}
1090
1091static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
 
1092				      int oob_required, int page)
1093{
1094	struct mtd_info *mtd = nand_to_mtd(nand);
1095	struct nand_ecc_ctrl *ecc = &nand->ecc;
1096	unsigned int max_bitflips = 0;
1097	int ret, i, cur_off = 0;
1098	bool raw_mode = false;
1099
1100	sunxi_nfc_select_chip(nand, nand->cur_cs);
1101
1102	nand_read_page_op(nand, page, 0, NULL, 0);
1103
1104	sunxi_nfc_hw_ecc_enable(nand);
1105
1106	for (i = 0; i < ecc->steps; i++) {
1107		int data_off = i * ecc->size;
1108		int oob_off = i * (ecc->bytes + 4);
1109		u8 *data = buf + data_off;
1110		u8 *oob = nand->oob_poi + oob_off;
1111
1112		ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1113						  oob_off + mtd->writesize,
1114						  &cur_off, &max_bitflips,
1115						  !i, oob_required, page);
1116		if (ret < 0)
1117			return ret;
1118		else if (ret)
1119			raw_mode = true;
1120	}
1121
1122	if (oob_required)
1123		sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1124						!raw_mode, page);
1125
1126	sunxi_nfc_hw_ecc_disable(nand);
1127
1128	return max_bitflips;
1129}
1130
1131static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
 
1132					  int oob_required, int page)
1133{
1134	int ret;
1135
1136	sunxi_nfc_select_chip(nand, nand->cur_cs);
1137
1138	nand_read_page_op(nand, page, 0, NULL, 0);
1139
1140	ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1141					       nand->ecc.steps);
1142	if (ret >= 0)
1143		return ret;
1144
1145	/* Fallback to PIO mode */
1146	return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1147}
1148
1149static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
 
1150					 u32 data_offs, u32 readlen,
1151					 u8 *bufpoi, int page)
1152{
1153	struct mtd_info *mtd = nand_to_mtd(nand);
1154	struct nand_ecc_ctrl *ecc = &nand->ecc;
1155	int ret, i, cur_off = 0;
1156	unsigned int max_bitflips = 0;
1157
1158	sunxi_nfc_select_chip(nand, nand->cur_cs);
1159
1160	nand_read_page_op(nand, page, 0, NULL, 0);
1161
1162	sunxi_nfc_hw_ecc_enable(nand);
1163
1164	for (i = data_offs / ecc->size;
1165	     i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1166		int data_off = i * ecc->size;
1167		int oob_off = i * (ecc->bytes + 4);
1168		u8 *data = bufpoi + data_off;
1169		u8 *oob = nand->oob_poi + oob_off;
1170
1171		ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1172						  oob,
1173						  oob_off + mtd->writesize,
1174						  &cur_off, &max_bitflips, !i,
1175						  false, page);
1176		if (ret < 0)
1177			return ret;
1178	}
1179
1180	sunxi_nfc_hw_ecc_disable(nand);
1181
1182	return max_bitflips;
1183}
1184
1185static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
 
1186					     u32 data_offs, u32 readlen,
1187					     u8 *buf, int page)
1188{
1189	int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1190	int ret;
1191
1192	sunxi_nfc_select_chip(nand, nand->cur_cs);
1193
1194	nand_read_page_op(nand, page, 0, NULL, 0);
1195
1196	ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1197	if (ret >= 0)
1198		return ret;
1199
1200	/* Fallback to PIO mode */
1201	return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1202					     buf, page);
1203}
1204
1205static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
 
1206				       const uint8_t *buf, int oob_required,
1207				       int page)
1208{
1209	struct mtd_info *mtd = nand_to_mtd(nand);
1210	struct nand_ecc_ctrl *ecc = &nand->ecc;
1211	int ret, i, cur_off = 0;
1212
1213	sunxi_nfc_select_chip(nand, nand->cur_cs);
1214
1215	nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1216
1217	sunxi_nfc_hw_ecc_enable(nand);
1218
1219	for (i = 0; i < ecc->steps; i++) {
1220		int data_off = i * ecc->size;
1221		int oob_off = i * (ecc->bytes + 4);
1222		const u8 *data = buf + data_off;
1223		const u8 *oob = nand->oob_poi + oob_off;
1224
1225		ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1226						   oob_off + mtd->writesize,
1227						   &cur_off, !i, page);
1228		if (ret)
1229			return ret;
1230	}
1231
1232	if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1233		sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1234						 &cur_off, page);
1235
1236	sunxi_nfc_hw_ecc_disable(nand);
1237
1238	return nand_prog_page_end_op(nand);
1239}
1240
1241static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
 
1242					  u32 data_offs, u32 data_len,
1243					  const u8 *buf, int oob_required,
1244					  int page)
1245{
1246	struct mtd_info *mtd = nand_to_mtd(nand);
1247	struct nand_ecc_ctrl *ecc = &nand->ecc;
1248	int ret, i, cur_off = 0;
1249
1250	sunxi_nfc_select_chip(nand, nand->cur_cs);
1251
1252	nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1253
1254	sunxi_nfc_hw_ecc_enable(nand);
1255
1256	for (i = data_offs / ecc->size;
1257	     i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1258		int data_off = i * ecc->size;
1259		int oob_off = i * (ecc->bytes + 4);
1260		const u8 *data = buf + data_off;
1261		const u8 *oob = nand->oob_poi + oob_off;
1262
1263		ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1264						   oob_off + mtd->writesize,
1265						   &cur_off, !i, page);
1266		if (ret)
1267			return ret;
1268	}
1269
1270	sunxi_nfc_hw_ecc_disable(nand);
1271
1272	return nand_prog_page_end_op(nand);
1273}
1274
1275static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
 
1276					   const u8 *buf,
1277					   int oob_required,
1278					   int page)
1279{
 
1280	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1281	struct nand_ecc_ctrl *ecc = &nand->ecc;
1282	struct scatterlist sg;
1283	u32 wait;
1284	int ret, i;
1285
1286	sunxi_nfc_select_chip(nand, nand->cur_cs);
1287
1288	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1289	if (ret)
1290		return ret;
1291
1292	ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1293				       DMA_TO_DEVICE, &sg);
1294	if (ret)
1295		goto pio_fallback;
1296
1297	for (i = 0; i < ecc->steps; i++) {
1298		const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1299
1300		sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1301	}
1302
1303	nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1304
1305	sunxi_nfc_hw_ecc_enable(nand);
1306	sunxi_nfc_randomizer_config(nand, page, false);
1307	sunxi_nfc_randomizer_enable(nand);
1308
1309	writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1310	       nfc->regs + NFC_REG_WCMD_SET);
1311
1312	wait = NFC_CMD_INT_FLAG;
1313
1314	if (nfc->caps->has_mdma)
1315		wait |= NFC_DMA_INT_FLAG;
1316	else
1317		dma_async_issue_pending(nfc->dmac);
1318
1319	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1320	       NFC_DATA_TRANS | NFC_ACCESS_DIR,
1321	       nfc->regs + NFC_REG_CMD);
1322
1323	ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
1324	if (ret && !nfc->caps->has_mdma)
1325		dmaengine_terminate_all(nfc->dmac);
1326
1327	sunxi_nfc_randomizer_disable(nand);
1328	sunxi_nfc_hw_ecc_disable(nand);
1329
1330	sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1331
1332	if (ret)
1333		return ret;
1334
1335	if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1336		/* TODO: use DMA to transfer extra OOB bytes ? */
1337		sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1338						 NULL, page);
1339
1340	return nand_prog_page_end_op(nand);
1341
1342pio_fallback:
1343	return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1344}
1345
1346static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
 
 
1347{
1348	u8 *buf = nand_get_data_buf(nand);
1349
1350	return nand->ecc.read_page(nand, buf, 1, page);
1351}
1352
1353static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
 
 
1354{
1355	struct mtd_info *mtd = nand_to_mtd(nand);
1356	u8 *buf = nand_get_data_buf(nand);
1357	int ret;
1358
1359	memset(buf, 0xff, mtd->writesize);
1360	ret = nand->ecc.write_page(nand, buf, 1, page);
 
 
1361	if (ret)
1362		return ret;
1363
1364	/* Send command to program the OOB data */
1365	return nand_prog_page_end_op(nand);
1366}
1367
1368static const s32 tWB_lut[] = {6, 12, 16, 20};
1369static const s32 tRHW_lut[] = {4, 8, 12, 20};
1370
1371static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1372		u32 clk_period)
1373{
1374	u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1375	int i;
1376
1377	for (i = 0; i < lut_size; i++) {
1378		if (clk_cycles <= lut[i])
1379			return i;
1380	}
1381
1382	/* Doesn't fit */
1383	return -EINVAL;
1384}
1385
1386#define sunxi_nand_lookup_timing(l, p, c) \
1387			_sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1388
1389static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
1390				     const struct nand_interface_config *conf)
1391{
1392	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1393	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
 
1394	const struct nand_sdr_timings *timings;
1395	u32 min_clk_period = 0;
1396	s32 tWB, tADL, tWHR, tRHW, tCAD;
1397	long real_clk_rate;
1398
1399	timings = nand_get_sdr_timings(conf);
1400	if (IS_ERR(timings))
1401		return -ENOTSUPP;
1402
1403	/* T1 <=> tCLS */
1404	if (timings->tCLS_min > min_clk_period)
1405		min_clk_period = timings->tCLS_min;
1406
1407	/* T2 <=> tCLH */
1408	if (timings->tCLH_min > min_clk_period)
1409		min_clk_period = timings->tCLH_min;
1410
1411	/* T3 <=> tCS */
1412	if (timings->tCS_min > min_clk_period)
1413		min_clk_period = timings->tCS_min;
1414
1415	/* T4 <=> tCH */
1416	if (timings->tCH_min > min_clk_period)
1417		min_clk_period = timings->tCH_min;
1418
1419	/* T5 <=> tWP */
1420	if (timings->tWP_min > min_clk_period)
1421		min_clk_period = timings->tWP_min;
1422
1423	/* T6 <=> tWH */
1424	if (timings->tWH_min > min_clk_period)
1425		min_clk_period = timings->tWH_min;
1426
1427	/* T7 <=> tALS */
1428	if (timings->tALS_min > min_clk_period)
1429		min_clk_period = timings->tALS_min;
1430
1431	/* T8 <=> tDS */
1432	if (timings->tDS_min > min_clk_period)
1433		min_clk_period = timings->tDS_min;
1434
1435	/* T9 <=> tDH */
1436	if (timings->tDH_min > min_clk_period)
1437		min_clk_period = timings->tDH_min;
1438
1439	/* T10 <=> tRR */
1440	if (timings->tRR_min > (min_clk_period * 3))
1441		min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1442
1443	/* T11 <=> tALH */
1444	if (timings->tALH_min > min_clk_period)
1445		min_clk_period = timings->tALH_min;
1446
1447	/* T12 <=> tRP */
1448	if (timings->tRP_min > min_clk_period)
1449		min_clk_period = timings->tRP_min;
1450
1451	/* T13 <=> tREH */
1452	if (timings->tREH_min > min_clk_period)
1453		min_clk_period = timings->tREH_min;
1454
1455	/* T14 <=> tRC */
1456	if (timings->tRC_min > (min_clk_period * 2))
1457		min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1458
1459	/* T15 <=> tWC */
1460	if (timings->tWC_min > (min_clk_period * 2))
1461		min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1462
1463	/* T16 - T19 + tCAD */
1464	if (timings->tWB_max > (min_clk_period * 20))
1465		min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1466
1467	if (timings->tADL_min > (min_clk_period * 32))
1468		min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1469
1470	if (timings->tWHR_min > (min_clk_period * 32))
1471		min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1472
1473	if (timings->tRHW_min > (min_clk_period * 20))
1474		min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1475
1476	/*
1477	 * In non-EDO, tREA should be less than tRP to guarantee that the
1478	 * controller does not sample the IO lines too early. Unfortunately,
1479	 * the sunxi NAND controller does not allow us to have different
1480	 * values for tRP and tREH (tRP = tREH = tRW / 2).
1481	 *
1482	 * We have 2 options to overcome this limitation:
1483	 *
1484	 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1485	 * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1486	 */
1487	if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1488		min_clk_period = timings->tREA_max;
1489
1490	tWB  = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1491					min_clk_period);
1492	if (tWB < 0) {
1493		dev_err(nfc->dev, "unsupported tWB\n");
1494		return tWB;
1495	}
1496
1497	tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1498	if (tADL > 3) {
1499		dev_err(nfc->dev, "unsupported tADL\n");
1500		return -EINVAL;
1501	}
1502
1503	tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1504	if (tWHR > 3) {
1505		dev_err(nfc->dev, "unsupported tWHR\n");
1506		return -EINVAL;
1507	}
1508
1509	tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1510					min_clk_period);
1511	if (tRHW < 0) {
1512		dev_err(nfc->dev, "unsupported tRHW\n");
1513		return tRHW;
1514	}
1515
1516	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1517		return 0;
1518
1519	/*
1520	 * TODO: according to ONFI specs this value only applies for DDR NAND,
1521	 * but Allwinner seems to set this to 0x7. Mimic them for now.
1522	 */
1523	tCAD = 0x7;
1524
1525	/* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1526	sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1527
1528	/* Convert min_clk_period from picoseconds to nanoseconds */
1529	min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1530
1531	/*
1532	 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1533	 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1534	 * This new formula was verified with a scope and validated by
1535	 * Allwinner engineers.
1536	 */
1537	sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1538	real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1539	if (real_clk_rate <= 0) {
1540		dev_err(nfc->dev, "Unable to round clk %lu\n",
1541			sunxi_nand->clk_rate);
1542		return -EINVAL;
1543	}
1544
1545	sunxi_nand->timing_ctl = 0;
1546
1547	/*
1548	 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1549	 * output cycle timings shall be used if the host drives tRC less than
1550	 * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1551	 */
1552	min_clk_period = NSEC_PER_SEC / real_clk_rate;
1553	if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1554		sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1555
1556	return 0;
1557}
1558
1559static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1560				    struct mtd_oob_region *oobregion)
1561{
1562	struct nand_chip *nand = mtd_to_nand(mtd);
1563	struct nand_ecc_ctrl *ecc = &nand->ecc;
1564
1565	if (section >= ecc->steps)
1566		return -ERANGE;
1567
1568	oobregion->offset = section * (ecc->bytes + 4) + 4;
1569	oobregion->length = ecc->bytes;
1570
1571	return 0;
1572}
1573
1574static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1575				     struct mtd_oob_region *oobregion)
1576{
1577	struct nand_chip *nand = mtd_to_nand(mtd);
1578	struct nand_ecc_ctrl *ecc = &nand->ecc;
1579
1580	if (section > ecc->steps)
1581		return -ERANGE;
1582
1583	/*
1584	 * The first 2 bytes are used for BB markers, hence we
1585	 * only have 2 bytes available in the first user data
1586	 * section.
1587	 */
1588	if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
1589		oobregion->offset = 2;
1590		oobregion->length = 2;
1591
1592		return 0;
1593	}
1594
1595	/*
1596	 * The controller does not provide access to OOB bytes
1597	 * past the end of the ECC data.
1598	 */
1599	if (section == ecc->steps && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST)
1600		return -ERANGE;
1601
1602	oobregion->offset = section * (ecc->bytes + 4);
1603
1604	if (section < ecc->steps)
1605		oobregion->length = 4;
1606	else
1607		oobregion->length = mtd->oobsize - oobregion->offset;
1608
1609	return 0;
1610}
1611
1612static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1613	.ecc = sunxi_nand_ooblayout_ecc,
1614	.free = sunxi_nand_ooblayout_free,
1615};
1616
1617static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
 
 
 
 
 
1618				       struct nand_ecc_ctrl *ecc,
1619				       struct device_node *np)
1620{
1621	static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
 
1622	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1623	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1624	struct mtd_info *mtd = nand_to_mtd(nand);
1625	struct nand_device *nanddev = mtd_to_nanddev(mtd);
1626	int nsectors;
 
1627	int i;
1628
1629	if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
1630		int bytes;
1631
1632		ecc->size = 1024;
1633		nsectors = mtd->writesize / ecc->size;
1634
1635		/* Reserve 2 bytes for the BBM */
1636		bytes = (mtd->oobsize - 2) / nsectors;
1637
1638		/* 4 non-ECC bytes are added before each ECC bytes section */
1639		bytes -= 4;
1640
1641		/* and bytes has to be even. */
1642		if (bytes % 2)
1643			bytes--;
1644
1645		ecc->strength = bytes * 8 / fls(8 * ecc->size);
1646
1647		for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1648			if (strengths[i] > ecc->strength)
1649				break;
1650		}
1651
1652		if (!i)
1653			ecc->strength = 0;
1654		else
1655			ecc->strength = strengths[i - 1];
1656	}
1657
1658	if (ecc->size != 512 && ecc->size != 1024)
1659		return -EINVAL;
1660
 
 
 
 
1661	/* Prefer 1k ECC chunk over 512 ones */
1662	if (ecc->size == 512 && mtd->writesize > 512) {
1663		ecc->size = 1024;
1664		ecc->strength *= 2;
1665	}
1666
1667	/* Add ECC info retrieval from DT */
1668	for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1669		if (ecc->strength <= strengths[i]) {
1670			/*
1671			 * Update ecc->strength value with the actual strength
1672			 * that will be used by the ECC engine.
1673			 */
1674			ecc->strength = strengths[i];
1675			break;
1676		}
1677	}
1678
1679	if (i >= ARRAY_SIZE(strengths)) {
1680		dev_err(nfc->dev, "unsupported strength\n");
1681		return -ENOTSUPP;
 
1682	}
1683
 
 
1684	/* HW ECC always request ECC bytes for 1024 bytes blocks */
1685	ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1686
1687	/* HW ECC always work with even numbers of ECC bytes */
1688	ecc->bytes = ALIGN(ecc->bytes, 2);
1689
1690	nsectors = mtd->writesize / ecc->size;
1691
1692	if (mtd->oobsize < ((ecc->bytes + 4) * nsectors))
1693		return -EINVAL;
 
 
1694
1695	ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1696	ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1697	mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
 
1698
1699	if (nfc->dmac || nfc->caps->has_mdma) {
1700		ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1701		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1702		ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1703		nand->options |= NAND_USES_DMA;
1704	} else {
1705		ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1706		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1707		ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1708	}
1709
1710	/* TODO: support DMA for raw accesses and subpage write */
1711	ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1712	ecc->read_oob_raw = nand_read_oob_std;
1713	ecc->write_oob_raw = nand_write_oob_std;
1714
1715	sunxi_nand->ecc.ecc_ctl = NFC_ECC_MODE(i) | NFC_ECC_EXCEPTION |
1716				  NFC_ECC_PIPELINE | NFC_ECC_EN;
1717
1718	if (ecc->size == 512)
1719		sunxi_nand->ecc.ecc_ctl |= NFC_ECC_BLOCK_512;
1720
1721	return 0;
1722}
1723
1724static int sunxi_nand_attach_chip(struct nand_chip *nand)
1725{
1726	const struct nand_ecc_props *requirements =
1727		nanddev_get_ecc_requirements(&nand->base);
1728	struct nand_ecc_ctrl *ecc = &nand->ecc;
1729	struct device_node *np = nand_get_flash_node(nand);
1730	int ret;
1731
1732	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1733		nand->bbt_options |= NAND_BBT_NO_OOB;
1734
1735	if (nand->options & NAND_NEED_SCRAMBLING)
1736		nand->options |= NAND_NO_SUBPAGE_WRITE;
1737
1738	nand->options |= NAND_SUBPAGE_READ;
 
 
 
 
1739
1740	if (!ecc->size) {
1741		ecc->size = requirements->step_size;
1742		ecc->strength = requirements->strength;
1743	}
1744
1745	if (!ecc->size || !ecc->strength)
1746		return -EINVAL;
1747
1748	switch (ecc->engine_type) {
1749	case NAND_ECC_ENGINE_TYPE_ON_HOST:
1750		ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1751		if (ret)
1752			return ret;
1753		break;
1754	case NAND_ECC_ENGINE_TYPE_NONE:
1755	case NAND_ECC_ENGINE_TYPE_SOFT:
1756		break;
1757	default:
1758		return -EINVAL;
1759	}
1760
1761	return 0;
1762}
1763
1764static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1765				const struct nand_subop *subop)
1766{
1767	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1768	u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1769	unsigned int i, j, remaining, start;
1770	void *inbuf = NULL;
1771	int ret;
1772
1773	for (i = 0; i < subop->ninstrs; i++) {
1774		const struct nand_op_instr *instr = &subop->instrs[i];
1775
1776		switch (instr->type) {
1777		case NAND_OP_CMD_INSTR:
1778			if (cmd & NFC_SEND_CMD1) {
1779				if (WARN_ON(cmd & NFC_SEND_CMD2))
1780					return -EINVAL;
1781
1782				cmd |= NFC_SEND_CMD2;
1783				extcmd |= instr->ctx.cmd.opcode;
1784			} else {
1785				cmd |= NFC_SEND_CMD1 |
1786				       NFC_CMD(instr->ctx.cmd.opcode);
1787			}
1788			break;
1789
1790		case NAND_OP_ADDR_INSTR:
1791			remaining = nand_subop_get_num_addr_cyc(subop, i);
1792			start = nand_subop_get_addr_start_off(subop, i);
1793			for (j = 0; j < 8 && j + start < remaining; j++) {
1794				u32 addr = instr->ctx.addr.addrs[j + start];
1795
1796				addrs[j / 4] |= addr << (j % 4) * 8;
1797			}
1798
1799			if (j)
1800				cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1801
1802			break;
1803
1804		case NAND_OP_DATA_IN_INSTR:
1805		case NAND_OP_DATA_OUT_INSTR:
1806			start = nand_subop_get_data_start_off(subop, i);
1807			remaining = nand_subop_get_data_len(subop, i);
1808			cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1809			cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1810
1811			if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1812				cmd |= NFC_ACCESS_DIR;
1813				memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1814					    instr->ctx.data.buf.out + start,
1815					    cnt);
1816			} else {
1817				inbuf = instr->ctx.data.buf.in + start;
1818			}
1819
1820			break;
1821
1822		case NAND_OP_WAITRDY_INSTR:
1823			cmd |= NFC_WAIT_FLAG;
1824			break;
1825		}
1826	}
1827
1828	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1829	if (ret)
1830		return ret;
1831
1832	if (cmd & NFC_SEND_ADR) {
1833		writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1834		writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1835	}
1836
1837	if (cmd & NFC_SEND_CMD2)
1838		writel(extcmd,
1839		       nfc->regs +
1840		       (cmd & NFC_ACCESS_DIR ?
1841			NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1842
1843	if (cmd & NFC_DATA_TRANS)
1844		writel(cnt, nfc->regs + NFC_REG_CNT);
1845
1846	writel(cmd, nfc->regs + NFC_REG_CMD);
1847
1848	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1849				    !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1850				    0);
1851	if (ret)
1852		return ret;
1853
1854	if (inbuf)
1855		memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1856
1857	return 0;
1858}
1859
1860static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1861				  const struct nand_subop *subop)
1862{
1863	return nand_soft_waitrdy(nand,
1864				 subop->instrs[0].ctx.waitrdy.timeout_ms);
1865}
1866
1867static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1868	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1869			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870			       NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1871			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1872			       NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1873			       NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1874	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1875			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1876			       NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1877			       NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1878			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1879			       NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1880);
1881
1882static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1883	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1884			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1885			       NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1886			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1887			       NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1888	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1889			       NAND_OP_PARSER_PAT_CMD_ELEM(true),
1890			       NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1891			       NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1892			       NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1893	NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1894			       NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1895);
1896
1897static int sunxi_nfc_exec_op(struct nand_chip *nand,
1898			     const struct nand_operation *op, bool check_only)
1899{
1900	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1901	const struct nand_op_parser *parser;
1902
1903	if (!check_only)
1904		sunxi_nfc_select_chip(nand, op->cs);
1905
1906	if (sunxi_nand->sels[op->cs].rb >= 0)
1907		parser = &sunxi_nfc_op_parser;
1908	else
1909		parser = &sunxi_nfc_norb_op_parser;
1910
1911	return nand_op_parser_exec_op(nand, parser, op, check_only);
1912}
1913
1914static const struct nand_controller_ops sunxi_nand_controller_ops = {
1915	.attach_chip = sunxi_nand_attach_chip,
1916	.setup_interface = sunxi_nfc_setup_interface,
1917	.exec_op = sunxi_nfc_exec_op,
1918};
1919
1920static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1921{
1922	struct sunxi_nand_chip *sunxi_nand;
1923	struct nand_chip *chip;
1924	int ret;
1925
1926	while (!list_empty(&nfc->chips)) {
1927		sunxi_nand = list_first_entry(&nfc->chips,
1928					      struct sunxi_nand_chip,
1929					      node);
1930		chip = &sunxi_nand->nand;
1931		ret = mtd_device_unregister(nand_to_mtd(chip));
1932		WARN_ON(ret);
1933		nand_cleanup(chip);
1934		list_del(&sunxi_nand->node);
1935	}
1936}
1937
1938static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1939				struct device_node *np)
1940{
1941	struct sunxi_nand_chip *sunxi_nand;
1942	struct mtd_info *mtd;
1943	struct nand_chip *nand;
1944	int nsels;
1945	int ret;
1946	int i;
1947	u32 tmp;
1948
1949	if (!of_get_property(np, "reg", &nsels))
1950		return -EINVAL;
1951
1952	nsels /= sizeof(u32);
1953	if (!nsels) {
1954		dev_err(dev, "invalid reg property size\n");
1955		return -EINVAL;
1956	}
1957
1958	sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1959				  GFP_KERNEL);
1960	if (!sunxi_nand)
 
 
 
1961		return -ENOMEM;
 
1962
1963	sunxi_nand->nsels = nsels;
 
1964
1965	for (i = 0; i < nsels; i++) {
1966		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1967		if (ret) {
1968			dev_err(dev, "could not retrieve reg property: %d\n",
1969				ret);
1970			return ret;
1971		}
1972
1973		if (tmp > NFC_MAX_CS) {
1974			dev_err(dev,
1975				"invalid reg value: %u (max CS = 7)\n",
1976				tmp);
1977			return -EINVAL;
1978		}
1979
1980		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1981			dev_err(dev, "CS %d already assigned\n", tmp);
1982			return -EINVAL;
1983		}
1984
1985		sunxi_nand->sels[i].cs = tmp;
1986
1987		if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1988		    tmp < 2)
1989			sunxi_nand->sels[i].rb = tmp;
1990		else
1991			sunxi_nand->sels[i].rb = -1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1992	}
1993
1994	nand = &sunxi_nand->nand;
1995	/* Default tR value specified in the ONFI spec (chapter 4.15.1) */
 
1996	nand->controller = &nfc->controller;
1997	nand->controller->ops = &sunxi_nand_controller_ops;
1998
1999	/*
2000	 * Set the ECC mode to the default value in case nothing is specified
2001	 * in the DT.
2002	 */
2003	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2004	nand_set_flash_node(nand, np);
 
 
 
 
 
 
2005
2006	mtd = nand_to_mtd(nand);
2007	mtd->dev.parent = dev;
2008
2009	ret = nand_scan(nand, nsels);
2010	if (ret)
2011		return ret;
2012
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2013	ret = mtd_device_register(mtd, NULL, 0);
2014	if (ret) {
2015		dev_err(dev, "failed to register mtd device: %d\n", ret);
2016		nand_cleanup(nand);
2017		return ret;
2018	}
2019
2020	list_add_tail(&sunxi_nand->node, &nfc->chips);
2021
2022	return 0;
2023}
2024
2025static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2026{
2027	struct device_node *np = dev->of_node;
2028	struct device_node *nand_np;
 
2029	int ret;
2030
 
 
 
 
 
2031	for_each_child_of_node(np, nand_np) {
2032		ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2033		if (ret) {
2034			of_node_put(nand_np);
2035			sunxi_nand_chips_cleanup(nfc);
2036			return ret;
2037		}
2038	}
2039
2040	return 0;
2041}
2042
2043static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)
2044{
2045	int ret;
2046
2047	if (nfc->caps->has_mdma)
2048		return 0;
2049
2050	nfc->dmac = dma_request_chan(nfc->dev, "rxtx");
2051	if (IS_ERR(nfc->dmac)) {
2052		ret = PTR_ERR(nfc->dmac);
2053		if (ret == -EPROBE_DEFER)
2054			return ret;
2055
2056		/* Ignore errors to fall back to PIO mode */
2057		dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);
2058		nfc->dmac = NULL;
2059	} else {
2060		struct dma_slave_config dmac_cfg = { };
2061
2062		dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2063		dmac_cfg.dst_addr = dmac_cfg.src_addr;
2064		dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2065		dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2066		dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2067		dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2068		dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2069	}
2070	return 0;
2071}
2072
2073static int sunxi_nfc_probe(struct platform_device *pdev)
2074{
2075	struct device *dev = &pdev->dev;
2076	struct resource *r;
2077	struct sunxi_nfc *nfc;
2078	int irq;
2079	int ret;
2080
2081	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2082	if (!nfc)
2083		return -ENOMEM;
2084
2085	nfc->dev = dev;
2086	nand_controller_init(&nfc->controller);
2087	INIT_LIST_HEAD(&nfc->chips);
2088
2089	nfc->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
 
2090	if (IS_ERR(nfc->regs))
2091		return PTR_ERR(nfc->regs);
2092
2093	irq = platform_get_irq(pdev, 0);
2094	if (irq < 0)
 
2095		return irq;
 
2096
2097	nfc->ahb_clk = devm_clk_get_enabled(dev, "ahb");
2098	if (IS_ERR(nfc->ahb_clk)) {
2099		dev_err(dev, "failed to retrieve ahb clk\n");
2100		return PTR_ERR(nfc->ahb_clk);
2101	}
2102
2103	nfc->mod_clk = devm_clk_get_enabled(dev, "mod");
 
 
 
 
2104	if (IS_ERR(nfc->mod_clk)) {
2105		dev_err(dev, "failed to retrieve mod clk\n");
2106		return PTR_ERR(nfc->mod_clk);
 
2107	}
2108
 
 
 
 
2109	nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2110	if (IS_ERR(nfc->reset))
2111		return PTR_ERR(nfc->reset);
 
 
2112
2113	ret = reset_control_deassert(nfc->reset);
2114	if (ret) {
2115		dev_err(dev, "reset err %d\n", ret);
2116		return ret;
2117	}
2118
2119	nfc->caps = of_device_get_match_data(&pdev->dev);
2120	if (!nfc->caps) {
2121		ret = -EINVAL;
2122		goto out_ahb_reset_reassert;
2123	}
2124
2125	ret = sunxi_nfc_rst(nfc);
2126	if (ret)
2127		goto out_ahb_reset_reassert;
2128
2129	writel(0, nfc->regs + NFC_REG_INT);
2130	ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2131			       0, "sunxi-nand", nfc);
2132	if (ret)
2133		goto out_ahb_reset_reassert;
2134
2135	ret = sunxi_nfc_dma_init(nfc, r);
 
 
2136
2137	if (ret)
2138		goto out_ahb_reset_reassert;
 
 
 
 
 
 
 
 
2139
2140	platform_set_drvdata(pdev, nfc);
2141
2142	ret = sunxi_nand_chips_init(dev, nfc);
2143	if (ret) {
2144		dev_err(dev, "failed to init nand chips\n");
2145		goto out_release_dmac;
2146	}
2147
2148	return 0;
2149
2150out_release_dmac:
2151	if (nfc->dmac)
2152		dma_release_channel(nfc->dmac);
2153out_ahb_reset_reassert:
2154	reset_control_assert(nfc->reset);
 
 
 
 
2155
2156	return ret;
2157}
2158
2159static void sunxi_nfc_remove(struct platform_device *pdev)
2160{
2161	struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2162
2163	sunxi_nand_chips_cleanup(nfc);
2164
2165	reset_control_assert(nfc->reset);
2166
2167	if (nfc->dmac)
2168		dma_release_channel(nfc->dmac);
2169}
 
2170
2171static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2172	.reg_io_data = NFC_REG_A10_IO_DATA,
2173	.dma_maxburst = 4,
2174};
2175
2176static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
2177	.has_mdma = true,
2178	.reg_io_data = NFC_REG_A23_IO_DATA,
2179	.dma_maxburst = 8,
2180};
2181
2182static const struct of_device_id sunxi_nfc_ids[] = {
2183	{
2184		.compatible = "allwinner,sun4i-a10-nand",
2185		.data = &sunxi_nfc_a10_caps,
2186	},
2187	{
2188		.compatible = "allwinner,sun8i-a23-nand-controller",
2189		.data = &sunxi_nfc_a23_caps,
2190	},
2191	{ /* sentinel */ }
2192};
2193MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2194
2195static struct platform_driver sunxi_nfc_driver = {
2196	.driver = {
2197		.name = "sunxi_nand",
2198		.of_match_table = sunxi_nfc_ids,
2199	},
2200	.probe = sunxi_nfc_probe,
2201	.remove_new = sunxi_nfc_remove,
2202};
2203module_platform_driver(sunxi_nfc_driver);
2204
2205MODULE_LICENSE("GPL");
2206MODULE_AUTHOR("Boris BREZILLON");
2207MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2208MODULE_ALIAS("platform:sunxi_nand");