1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * MTK NAND Flash controller driver.
   4 * Copyright (C) 2016 MediaTek Inc.
   5 * Authors:	Xiaolei Li		<xiaolei.li@mediatek.com>
   6 *		Jorge Ramirez-Ortiz	<jorge.ramirez-ortiz@linaro.org>
   7 */
   8
   9#include <linux/platform_device.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/interrupt.h>
  12#include <linux/delay.h>
  13#include <linux/clk.h>
  14#include <linux/mtd/rawnand.h>
  15#include <linux/mtd/mtd.h>
  16#include <linux/module.h>
  17#include <linux/iopoll.h>
  18#include <linux/of.h>
  19#include <linux/mtd/nand-ecc-mtk.h>
  20
  21/* NAND controller register definition */
  22#define NFI_CNFG		(0x00)
  23#define		CNFG_AHB		BIT(0)
  24#define		CNFG_READ_EN		BIT(1)
  25#define		CNFG_DMA_BURST_EN	BIT(2)
  26#define		CNFG_BYTE_RW		BIT(6)
  27#define		CNFG_HW_ECC_EN		BIT(8)
  28#define		CNFG_AUTO_FMT_EN	BIT(9)
  29#define		CNFG_OP_CUST		(6 << 12)
  30#define NFI_PAGEFMT		(0x04)
  31#define		PAGEFMT_FDM_ECC_SHIFT	(12)
  32#define		PAGEFMT_FDM_SHIFT	(8)
  33#define		PAGEFMT_SEC_SEL_512	BIT(2)
  34#define		PAGEFMT_512_2K		(0)
  35#define		PAGEFMT_2K_4K		(1)
  36#define		PAGEFMT_4K_8K		(2)
  37#define		PAGEFMT_8K_16K		(3)
  38/* NFI control */
  39#define NFI_CON			(0x08)
  40#define		CON_FIFO_FLUSH		BIT(0)
  41#define		CON_NFI_RST		BIT(1)
  42#define		CON_BRD			BIT(8)  /* burst  read */
  43#define		CON_BWR			BIT(9)	/* burst  write */
  44#define		CON_SEC_SHIFT		(12)
  45/* Timming control register */
  46#define NFI_ACCCON		(0x0C)
  47#define NFI_INTR_EN		(0x10)
  48#define		INTR_AHB_DONE_EN	BIT(6)
  49#define NFI_INTR_STA		(0x14)
  50#define NFI_CMD			(0x20)
  51#define NFI_ADDRNOB		(0x30)
  52#define NFI_COLADDR		(0x34)
  53#define NFI_ROWADDR		(0x38)
  54#define NFI_STRDATA		(0x40)
  55#define		STAR_EN			(1)
  56#define		STAR_DE			(0)
  57#define NFI_CNRNB		(0x44)
  58#define NFI_DATAW		(0x50)
  59#define NFI_DATAR		(0x54)
  60#define NFI_PIO_DIRDY		(0x58)
  61#define		PIO_DI_RDY		(0x01)
  62#define NFI_STA			(0x60)
  63#define		STA_CMD			BIT(0)
  64#define		STA_ADDR		BIT(1)
  65#define		STA_BUSY		BIT(8)
  66#define		STA_EMP_PAGE		BIT(12)
  67#define		NFI_FSM_CUSTDATA	(0xe << 16)
  68#define		NFI_FSM_MASK		(0xf << 16)
  69#define NFI_ADDRCNTR		(0x70)
  70#define		CNTR_MASK		GENMASK(16, 12)
  71#define		ADDRCNTR_SEC_SHIFT	(12)
  72#define		ADDRCNTR_SEC(val) \
  73		(((val) & CNTR_MASK) >> ADDRCNTR_SEC_SHIFT)
  74#define NFI_STRADDR		(0x80)
  75#define NFI_BYTELEN		(0x84)
  76#define NFI_CSEL		(0x90)
  77#define NFI_FDML(x)		(0xA0 + (x) * sizeof(u32) * 2)
  78#define NFI_FDMM(x)		(0xA4 + (x) * sizeof(u32) * 2)
  79#define NFI_FDM_MAX_SIZE	(8)
  80#define NFI_FDM_MIN_SIZE	(1)
  81#define NFI_DEBUG_CON1		(0x220)
  82#define		STROBE_MASK		GENMASK(4, 3)
  83#define		STROBE_SHIFT		(3)
  84#define		MAX_STROBE_DLY		(3)
  85#define NFI_MASTER_STA		(0x224)
  86#define		MASTER_STA_MASK		(0x0FFF)
  87#define NFI_EMPTY_THRESH	(0x23C)
  88
  89#define MTK_NAME		"mtk-nand"
  90#define KB(x)			((x) * 1024UL)
  91#define MB(x)			(KB(x) * 1024UL)
  92
  93#define MTK_TIMEOUT		(500000)
  94#define MTK_RESET_TIMEOUT	(1000000)
  95#define MTK_NAND_MAX_NSELS	(2)
  96#define MTK_NFC_MIN_SPARE	(16)
  97#define ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt) \
  98	((tpoecs) << 28 | (tprecs) << 22 | (tc2r) << 16 | \
  99	(tw2r) << 12 | (twh) << 8 | (twst) << 4 | (trlt))
 100
 101struct mtk_nfc_caps {
 102	const u8 *spare_size;
 103	u8 num_spare_size;
 104	u8 pageformat_spare_shift;
 105	u8 nfi_clk_div;
 106	u8 max_sector;
 107	u32 max_sector_size;
 108};
 109
 110struct mtk_nfc_bad_mark_ctl {
 111	void (*bm_swap)(struct mtd_info *, u8 *buf, int raw);
 112	u32 sec;
 113	u32 pos;
 114};
 115
 116/*
 117 * FDM: region used to store free OOB data
 118 */
 119struct mtk_nfc_fdm {
 120	u32 reg_size;
 121	u32 ecc_size;
 122};
 123
 124struct mtk_nfc_nand_chip {
 125	struct list_head node;
 126	struct nand_chip nand;
 127
 128	struct mtk_nfc_bad_mark_ctl bad_mark;
 129	struct mtk_nfc_fdm fdm;
 130	u32 spare_per_sector;
 131
 132	int nsels;
 133	u8 sels[] __counted_by(nsels);
 134	/* nothing after this field */
 135};
 136
 137struct mtk_nfc_clk {
 138	struct clk *nfi_clk;
 139	struct clk *pad_clk;
 140};
 141
 142struct mtk_nfc {
 143	struct nand_controller controller;
 144	struct mtk_ecc_config ecc_cfg;
 145	struct mtk_nfc_clk clk;
 146	struct mtk_ecc *ecc;
 147
 148	struct device *dev;
 149	const struct mtk_nfc_caps *caps;
 150	void __iomem *regs;
 151
 152	struct completion done;
 153	struct list_head chips;
 154
 155	u8 *buffer;
 156
 157	unsigned long assigned_cs;
 158};
 159
 160/*
 161 * supported spare size of each IP.
 162 * order should be the same with the spare size bitfiled defination of
 163 * register NFI_PAGEFMT.
 164 */
 165static const u8 spare_size_mt2701[] = {
 166	16, 26, 27, 28, 32, 36, 40, 44,	48, 49, 50, 51, 52, 62, 63, 64
 167};
 168
 169static const u8 spare_size_mt2712[] = {
 170	16, 26, 27, 28, 32, 36, 40, 44, 48, 49, 50, 51, 52, 62, 61, 63, 64, 67,
 171	74
 172};
 173
 174static const u8 spare_size_mt7622[] = {
 175	16, 26, 27, 28
 176};
 177
 178static inline struct mtk_nfc_nand_chip *to_mtk_nand(struct nand_chip *nand)
 179{
 180	return container_of(nand, struct mtk_nfc_nand_chip, nand);
 181}
 182
 183static inline u8 *data_ptr(struct nand_chip *chip, const u8 *p, int i)
 184{
 185	return (u8 *)p + i * chip->ecc.size;
 186}
 187
 188static inline u8 *oob_ptr(struct nand_chip *chip, int i)
 189{
 190	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 191	u8 *poi;
 192
 193	/* map the sector's FDM data to free oob:
 194	 * the beginning of the oob area stores the FDM data of bad mark sectors
 195	 */
 196
 197	if (i < mtk_nand->bad_mark.sec)
 198		poi = chip->oob_poi + (i + 1) * mtk_nand->fdm.reg_size;
 199	else if (i == mtk_nand->bad_mark.sec)
 200		poi = chip->oob_poi;
 201	else
 202		poi = chip->oob_poi + i * mtk_nand->fdm.reg_size;
 203
 204	return poi;
 205}
 206
 207static inline int mtk_data_len(struct nand_chip *chip)
 208{
 209	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 210
 211	return chip->ecc.size + mtk_nand->spare_per_sector;
 212}
 213
 214static inline u8 *mtk_data_ptr(struct nand_chip *chip,  int i)
 215{
 216	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 217
 218	return nfc->buffer + i * mtk_data_len(chip);
 219}
 220
 221static inline u8 *mtk_oob_ptr(struct nand_chip *chip, int i)
 222{
 223	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 224
 225	return nfc->buffer + i * mtk_data_len(chip) + chip->ecc.size;
 226}
 227
 228static inline void nfi_writel(struct mtk_nfc *nfc, u32 val, u32 reg)
 229{
 230	writel(val, nfc->regs + reg);
 231}
 232
 233static inline void nfi_writew(struct mtk_nfc *nfc, u16 val, u32 reg)
 234{
 235	writew(val, nfc->regs + reg);
 236}
 237
 238static inline void nfi_writeb(struct mtk_nfc *nfc, u8 val, u32 reg)
 239{
 240	writeb(val, nfc->regs + reg);
 241}
 242
 243static inline u32 nfi_readl(struct mtk_nfc *nfc, u32 reg)
 244{
 245	return readl_relaxed(nfc->regs + reg);
 246}
 247
 248static inline u16 nfi_readw(struct mtk_nfc *nfc, u32 reg)
 249{
 250	return readw_relaxed(nfc->regs + reg);
 251}
 252
 253static inline u8 nfi_readb(struct mtk_nfc *nfc, u32 reg)
 254{
 255	return readb_relaxed(nfc->regs + reg);
 256}
 257
 258static void mtk_nfc_hw_reset(struct mtk_nfc *nfc)
 259{
 260	struct device *dev = nfc->dev;
 261	u32 val;
 262	int ret;
 263
 264	/* reset all registers and force the NFI master to terminate */
 265	nfi_writel(nfc, CON_FIFO_FLUSH | CON_NFI_RST, NFI_CON);
 266
 267	/* wait for the master to finish the last transaction */
 268	ret = readl_poll_timeout(nfc->regs + NFI_MASTER_STA, val,
 269				 !(val & MASTER_STA_MASK), 50,
 270				 MTK_RESET_TIMEOUT);
 271	if (ret)
 272		dev_warn(dev, "master active in reset [0x%x] = 0x%x\n",
 273			 NFI_MASTER_STA, val);
 274
 275	/* ensure any status register affected by the NFI master is reset */
 276	nfi_writel(nfc, CON_FIFO_FLUSH | CON_NFI_RST, NFI_CON);
 277	nfi_writew(nfc, STAR_DE, NFI_STRDATA);
 278}
 279
 280static int mtk_nfc_send_command(struct mtk_nfc *nfc, u8 command)
 281{
 282	struct device *dev = nfc->dev;
 283	u32 val;
 284	int ret;
 285
 286	nfi_writel(nfc, command, NFI_CMD);
 287
 288	ret = readl_poll_timeout_atomic(nfc->regs + NFI_STA, val,
 289					!(val & STA_CMD), 10,  MTK_TIMEOUT);
 290	if (ret) {
 291		dev_warn(dev, "nfi core timed out entering command mode\n");
 292		return -EIO;
 293	}
 294
 295	return 0;
 296}
 297
 298static int mtk_nfc_send_address(struct mtk_nfc *nfc, int addr)
 299{
 300	struct device *dev = nfc->dev;
 301	u32 val;
 302	int ret;
 303
 304	nfi_writel(nfc, addr, NFI_COLADDR);
 305	nfi_writel(nfc, 0, NFI_ROWADDR);
 306	nfi_writew(nfc, 1, NFI_ADDRNOB);
 307
 308	ret = readl_poll_timeout_atomic(nfc->regs + NFI_STA, val,
 309					!(val & STA_ADDR), 10, MTK_TIMEOUT);
 310	if (ret) {
 311		dev_warn(dev, "nfi core timed out entering address mode\n");
 312		return -EIO;
 313	}
 314
 315	return 0;
 316}
 317
 318static int mtk_nfc_hw_runtime_config(struct mtd_info *mtd)
 319{
 320	struct nand_chip *chip = mtd_to_nand(mtd);
 321	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 322	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 323	u32 fmt, spare, i;
 324
 325	if (!mtd->writesize)
 326		return 0;
 327
 328	spare = mtk_nand->spare_per_sector;
 329
 330	switch (mtd->writesize) {
 331	case 512:
 332		fmt = PAGEFMT_512_2K | PAGEFMT_SEC_SEL_512;
 333		break;
 334	case KB(2):
 335		if (chip->ecc.size == 512)
 336			fmt = PAGEFMT_2K_4K | PAGEFMT_SEC_SEL_512;
 337		else
 338			fmt = PAGEFMT_512_2K;
 339		break;
 340	case KB(4):
 341		if (chip->ecc.size == 512)
 342			fmt = PAGEFMT_4K_8K | PAGEFMT_SEC_SEL_512;
 343		else
 344			fmt = PAGEFMT_2K_4K;
 345		break;
 346	case KB(8):
 347		if (chip->ecc.size == 512)
 348			fmt = PAGEFMT_8K_16K | PAGEFMT_SEC_SEL_512;
 349		else
 350			fmt = PAGEFMT_4K_8K;
 351		break;
 352	case KB(16):
 353		fmt = PAGEFMT_8K_16K;
 354		break;
 355	default:
 356		dev_err(nfc->dev, "invalid page len: %d\n", mtd->writesize);
 357		return -EINVAL;
 358	}
 359
 360	/*
 361	 * the hardware will double the value for this eccsize, so we need to
 362	 * halve it
 363	 */
 364	if (chip->ecc.size == 1024)
 365		spare >>= 1;
 366
 367	for (i = 0; i < nfc->caps->num_spare_size; i++) {
 368		if (nfc->caps->spare_size[i] == spare)
 369			break;
 370	}
 371
 372	if (i == nfc->caps->num_spare_size) {
 373		dev_err(nfc->dev, "invalid spare size %d\n", spare);
 374		return -EINVAL;
 375	}
 376
 377	fmt |= i << nfc->caps->pageformat_spare_shift;
 378
 379	fmt |= mtk_nand->fdm.reg_size << PAGEFMT_FDM_SHIFT;
 380	fmt |= mtk_nand->fdm.ecc_size << PAGEFMT_FDM_ECC_SHIFT;
 381	nfi_writel(nfc, fmt, NFI_PAGEFMT);
 382
 383	nfc->ecc_cfg.strength = chip->ecc.strength;
 384	nfc->ecc_cfg.len = chip->ecc.size + mtk_nand->fdm.ecc_size;
 385
 386	return 0;
 387}
 388
 389static inline void mtk_nfc_wait_ioready(struct mtk_nfc *nfc)
 390{
 391	int rc;
 392	u8 val;
 393
 394	rc = readb_poll_timeout_atomic(nfc->regs + NFI_PIO_DIRDY, val,
 395				       val & PIO_DI_RDY, 10, MTK_TIMEOUT);
 396	if (rc < 0)
 397		dev_err(nfc->dev, "data not ready\n");
 398}
 399
 400static inline u8 mtk_nfc_read_byte(struct nand_chip *chip)
 401{
 402	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 403	u32 reg;
 404
 405	/* after each byte read, the NFI_STA reg is reset by the hardware */
 406	reg = nfi_readl(nfc, NFI_STA) & NFI_FSM_MASK;
 407	if (reg != NFI_FSM_CUSTDATA) {
 408		reg = nfi_readw(nfc, NFI_CNFG);
 409		reg |= CNFG_BYTE_RW | CNFG_READ_EN;
 410		nfi_writew(nfc, reg, NFI_CNFG);
 411
 412		/*
 413		 * set to max sector to allow the HW to continue reading over
 414		 * unaligned accesses
 415		 */
 416		reg = (nfc->caps->max_sector << CON_SEC_SHIFT) | CON_BRD;
 417		nfi_writel(nfc, reg, NFI_CON);
 418
 419		/* trigger to fetch data */
 420		nfi_writew(nfc, STAR_EN, NFI_STRDATA);
 421	}
 422
 423	mtk_nfc_wait_ioready(nfc);
 424
 425	return nfi_readb(nfc, NFI_DATAR);
 426}
 427
 428static void mtk_nfc_read_buf(struct nand_chip *chip, u8 *buf, int len)
 429{
 430	int i;
 431
 432	for (i = 0; i < len; i++)
 433		buf[i] = mtk_nfc_read_byte(chip);
 434}
 435
 436static void mtk_nfc_write_byte(struct nand_chip *chip, u8 byte)
 437{
 438	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 439	u32 reg;
 440
 441	reg = nfi_readl(nfc, NFI_STA) & NFI_FSM_MASK;
 442
 443	if (reg != NFI_FSM_CUSTDATA) {
 444		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_BYTE_RW;
 445		nfi_writew(nfc, reg, NFI_CNFG);
 446
 447		reg = nfc->caps->max_sector << CON_SEC_SHIFT | CON_BWR;
 448		nfi_writel(nfc, reg, NFI_CON);
 449
 450		nfi_writew(nfc, STAR_EN, NFI_STRDATA);
 451	}
 452
 453	mtk_nfc_wait_ioready(nfc);
 454	nfi_writeb(nfc, byte, NFI_DATAW);
 455}
 456
 457static void mtk_nfc_write_buf(struct nand_chip *chip, const u8 *buf, int len)
 458{
 459	int i;
 460
 461	for (i = 0; i < len; i++)
 462		mtk_nfc_write_byte(chip, buf[i]);
 463}
 464
 465static int mtk_nfc_exec_instr(struct nand_chip *chip,
 466			      const struct nand_op_instr *instr)
 467{
 468	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 469	unsigned int i;
 470	u32 status;
 471
 472	switch (instr->type) {
 473	case NAND_OP_CMD_INSTR:
 474		mtk_nfc_send_command(nfc, instr->ctx.cmd.opcode);
 475		return 0;
 476	case NAND_OP_ADDR_INSTR:
 477		for (i = 0; i < instr->ctx.addr.naddrs; i++)
 478			mtk_nfc_send_address(nfc, instr->ctx.addr.addrs[i]);
 479		return 0;
 480	case NAND_OP_DATA_IN_INSTR:
 481		mtk_nfc_read_buf(chip, instr->ctx.data.buf.in,
 482				 instr->ctx.data.len);
 483		return 0;
 484	case NAND_OP_DATA_OUT_INSTR:
 485		mtk_nfc_write_buf(chip, instr->ctx.data.buf.out,
 486				  instr->ctx.data.len);
 487		return 0;
 488	case NAND_OP_WAITRDY_INSTR:
 489		return readl_poll_timeout(nfc->regs + NFI_STA, status,
 490					  !(status & STA_BUSY), 20,
 491					  instr->ctx.waitrdy.timeout_ms * 1000);
 492	default:
 493		break;
 494	}
 495
 496	return -EINVAL;
 497}
 498
 499static void mtk_nfc_select_target(struct nand_chip *nand, unsigned int cs)
 500{
 501	struct mtk_nfc *nfc = nand_get_controller_data(nand);
 502	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(nand);
 503
 504	mtk_nfc_hw_runtime_config(nand_to_mtd(nand));
 505
 506	nfi_writel(nfc, mtk_nand->sels[cs], NFI_CSEL);
 507}
 508
 509static int mtk_nfc_exec_op(struct nand_chip *chip,
 510			   const struct nand_operation *op,
 511			   bool check_only)
 512{
 513	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 514	unsigned int i;
 515	int ret = 0;
 516
 517	if (check_only)
 518		return 0;
 519
 520	mtk_nfc_hw_reset(nfc);
 521	nfi_writew(nfc, CNFG_OP_CUST, NFI_CNFG);
 522	mtk_nfc_select_target(chip, op->cs);
 523
 524	for (i = 0; i < op->ninstrs; i++) {
 525		ret = mtk_nfc_exec_instr(chip, &op->instrs[i]);
 526		if (ret)
 527			break;
 528	}
 529
 530	return ret;
 531}
 532
 533static int mtk_nfc_setup_interface(struct nand_chip *chip, int csline,
 534				   const struct nand_interface_config *conf)
 535{
 536	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 537	const struct nand_sdr_timings *timings;
 538	u32 rate, tpoecs, tprecs, tc2r, tw2r, twh, twst = 0, trlt = 0;
 539	u32 temp, tsel = 0;
 540
 541	timings = nand_get_sdr_timings(conf);
 542	if (IS_ERR(timings))
 543		return -ENOTSUPP;
 544
 545	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
 546		return 0;
 547
 548	rate = clk_get_rate(nfc->clk.nfi_clk);
 549	/* There is a frequency divider in some IPs */
 550	rate /= nfc->caps->nfi_clk_div;
 551
 552	/* turn clock rate into KHZ */
 553	rate /= 1000;
 554
 555	tpoecs = max(timings->tALH_min, timings->tCLH_min) / 1000;
 556	tpoecs = DIV_ROUND_UP(tpoecs * rate, 1000000);
 557	tpoecs &= 0xf;
 558
 559	tprecs = max(timings->tCLS_min, timings->tALS_min) / 1000;
 560	tprecs = DIV_ROUND_UP(tprecs * rate, 1000000);
 561	tprecs &= 0x3f;
 562
 563	/* sdr interface has no tCR which means CE# low to RE# low */
 564	tc2r = 0;
 565
 566	tw2r = timings->tWHR_min / 1000;
 567	tw2r = DIV_ROUND_UP(tw2r * rate, 1000000);
 568	tw2r = DIV_ROUND_UP(tw2r - 1, 2);
 569	tw2r &= 0xf;
 570
 571	twh = max(timings->tREH_min, timings->tWH_min) / 1000;
 572	twh = DIV_ROUND_UP(twh * rate, 1000000) - 1;
 573	twh &= 0xf;
 574
 575	/* Calculate real WE#/RE# hold time in nanosecond */
 576	temp = (twh + 1) * 1000000 / rate;
 577	/* nanosecond to picosecond */
 578	temp *= 1000;
 579
 580	/*
 581	 * WE# low level time should be expaned to meet WE# pulse time
 582	 * and WE# cycle time at the same time.
 583	 */
 584	if (temp < timings->tWC_min)
 585		twst = timings->tWC_min - temp;
 586	twst = max(timings->tWP_min, twst) / 1000;
 587	twst = DIV_ROUND_UP(twst * rate, 1000000) - 1;
 588	twst &= 0xf;
 589
 590	/*
 591	 * RE# low level time should be expaned to meet RE# pulse time
 592	 * and RE# cycle time at the same time.
 593	 */
 594	if (temp < timings->tRC_min)
 595		trlt = timings->tRC_min - temp;
 596	trlt = max(trlt, timings->tRP_min) / 1000;
 597	trlt = DIV_ROUND_UP(trlt * rate, 1000000) - 1;
 598	trlt &= 0xf;
 599
 600	/* Calculate RE# pulse time in nanosecond. */
 601	temp = (trlt + 1) * 1000000 / rate;
 602	/* nanosecond to picosecond */
 603	temp *= 1000;
 604	/*
 605	 * If RE# access time is bigger than RE# pulse time,
 606	 * delay sampling data timing.
 607	 */
 608	if (temp < timings->tREA_max) {
 609		tsel = timings->tREA_max / 1000;
 610		tsel = DIV_ROUND_UP(tsel * rate, 1000000);
 611		tsel -= (trlt + 1);
 612		if (tsel > MAX_STROBE_DLY) {
 613			trlt += tsel - MAX_STROBE_DLY;
 614			tsel = MAX_STROBE_DLY;
 615		}
 616	}
 617	temp = nfi_readl(nfc, NFI_DEBUG_CON1);
 618	temp &= ~STROBE_MASK;
 619	temp |= tsel << STROBE_SHIFT;
 620	nfi_writel(nfc, temp, NFI_DEBUG_CON1);
 621
 622	/*
 623	 * ACCON: access timing control register
 624	 * -------------------------------------
 625	 * 31:28: tpoecs, minimum required time for CS post pulling down after
 626	 *        accessing the device
 627	 * 27:22: tprecs, minimum required time for CS pre pulling down before
 628	 *        accessing the device
 629	 * 21:16: tc2r, minimum required time from NCEB low to NREB low
 630	 * 15:12: tw2r, minimum required time from NWEB high to NREB low.
 631	 * 11:08: twh, write enable hold time
 632	 * 07:04: twst, write wait states
 633	 * 03:00: trlt, read wait states
 634	 */
 635	trlt = ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt);
 636	nfi_writel(nfc, trlt, NFI_ACCCON);
 637
 638	return 0;
 639}
 640
 641static int mtk_nfc_sector_encode(struct nand_chip *chip, u8 *data)
 642{
 643	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 644	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 645	int size = chip->ecc.size + mtk_nand->fdm.reg_size;
 646
 647	nfc->ecc_cfg.mode = ECC_DMA_MODE;
 648	nfc->ecc_cfg.op = ECC_ENCODE;
 649
 650	return mtk_ecc_encode(nfc->ecc, &nfc->ecc_cfg, data, size);
 651}
 652
 653static void mtk_nfc_no_bad_mark_swap(struct mtd_info *a, u8 *b, int c)
 654{
 655	/* nop */
 656}
 657
 658static void mtk_nfc_bad_mark_swap(struct mtd_info *mtd, u8 *buf, int raw)
 659{
 660	struct nand_chip *chip = mtd_to_nand(mtd);
 661	struct mtk_nfc_nand_chip *nand = to_mtk_nand(chip);
 662	u32 bad_pos = nand->bad_mark.pos;
 663
 664	if (raw)
 665		bad_pos += nand->bad_mark.sec * mtk_data_len(chip);
 666	else
 667		bad_pos += nand->bad_mark.sec * chip->ecc.size;
 668
 669	swap(chip->oob_poi[0], buf[bad_pos]);
 670}
 671
 672static int mtk_nfc_format_subpage(struct mtd_info *mtd, u32 offset,
 673				  u32 len, const u8 *buf)
 674{
 675	struct nand_chip *chip = mtd_to_nand(mtd);
 676	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 677	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 678	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
 679	u32 start, end;
 680	int i, ret;
 681
 682	start = offset / chip->ecc.size;
 683	end = DIV_ROUND_UP(offset + len, chip->ecc.size);
 684
 685	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
 686	for (i = 0; i < chip->ecc.steps; i++) {
 687		memcpy(mtk_data_ptr(chip, i), data_ptr(chip, buf, i),
 688		       chip->ecc.size);
 689
 690		if (start > i || i >= end)
 691			continue;
 692
 693		if (i == mtk_nand->bad_mark.sec)
 694			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
 695
 696		memcpy(mtk_oob_ptr(chip, i), oob_ptr(chip, i), fdm->reg_size);
 697
 698		/* program the CRC back to the OOB */
 699		ret = mtk_nfc_sector_encode(chip, mtk_data_ptr(chip, i));
 700		if (ret < 0)
 701			return ret;
 702	}
 703
 704	return 0;
 705}
 706
 707static void mtk_nfc_format_page(struct mtd_info *mtd, const u8 *buf)
 708{
 709	struct nand_chip *chip = mtd_to_nand(mtd);
 710	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 711	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 712	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
 713	u32 i;
 714
 715	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
 716	for (i = 0; i < chip->ecc.steps; i++) {
 717		if (buf)
 718			memcpy(mtk_data_ptr(chip, i), data_ptr(chip, buf, i),
 719			       chip->ecc.size);
 720
 721		if (i == mtk_nand->bad_mark.sec)
 722			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
 723
 724		memcpy(mtk_oob_ptr(chip, i), oob_ptr(chip, i), fdm->reg_size);
 725	}
 726}
 727
 728static inline void mtk_nfc_read_fdm(struct nand_chip *chip, u32 start,
 729				    u32 sectors)
 730{
 731	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 732	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 733	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
 734	u32 vall, valm;
 735	u8 *oobptr;
 736	int i, j;
 737
 738	for (i = 0; i < sectors; i++) {
 739		oobptr = oob_ptr(chip, start + i);
 740		vall = nfi_readl(nfc, NFI_FDML(i));
 741		valm = nfi_readl(nfc, NFI_FDMM(i));
 742
 743		for (j = 0; j < fdm->reg_size; j++)
 744			oobptr[j] = (j >= 4 ? valm : vall) >> ((j % 4) * 8);
 745	}
 746}
 747
 748static inline void mtk_nfc_write_fdm(struct nand_chip *chip)
 749{
 750	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 751	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 752	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
 753	u32 vall, valm;
 754	u8 *oobptr;
 755	int i, j;
 756
 757	for (i = 0; i < chip->ecc.steps; i++) {
 758		oobptr = oob_ptr(chip, i);
 759		vall = 0;
 760		valm = 0;
 761		for (j = 0; j < 8; j++) {
 762			if (j < 4)
 763				vall |= (j < fdm->reg_size ? oobptr[j] : 0xff)
 764						<< (j * 8);
 765			else
 766				valm |= (j < fdm->reg_size ? oobptr[j] : 0xff)
 767						<< ((j - 4) * 8);
 768		}
 769		nfi_writel(nfc, vall, NFI_FDML(i));
 770		nfi_writel(nfc, valm, NFI_FDMM(i));
 771	}
 772}
 773
 774static int mtk_nfc_do_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 775				 const u8 *buf, int page, int len)
 776{
 777	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 778	struct device *dev = nfc->dev;
 779	dma_addr_t addr;
 780	u32 reg;
 781	int ret;
 782
 783	addr = dma_map_single(dev, (void *)buf, len, DMA_TO_DEVICE);
 784	ret = dma_mapping_error(nfc->dev, addr);
 785	if (ret) {
 786		dev_err(nfc->dev, "dma mapping error\n");
 787		return -EINVAL;
 788	}
 789
 790	reg = nfi_readw(nfc, NFI_CNFG) | CNFG_AHB | CNFG_DMA_BURST_EN;
 791	nfi_writew(nfc, reg, NFI_CNFG);
 792
 793	nfi_writel(nfc, chip->ecc.steps << CON_SEC_SHIFT, NFI_CON);
 794	nfi_writel(nfc, lower_32_bits(addr), NFI_STRADDR);
 795	nfi_writew(nfc, INTR_AHB_DONE_EN, NFI_INTR_EN);
 796
 797	init_completion(&nfc->done);
 798
 799	reg = nfi_readl(nfc, NFI_CON) | CON_BWR;
 800	nfi_writel(nfc, reg, NFI_CON);
 801	nfi_writew(nfc, STAR_EN, NFI_STRDATA);
 802
 803	ret = wait_for_completion_timeout(&nfc->done, msecs_to_jiffies(500));
 804	if (!ret) {
 805		dev_err(dev, "program ahb done timeout\n");
 806		nfi_writew(nfc, 0, NFI_INTR_EN);
 807		ret = -ETIMEDOUT;
 808		goto timeout;
 809	}
 810
 811	ret = readl_poll_timeout_atomic(nfc->regs + NFI_ADDRCNTR, reg,
 812					ADDRCNTR_SEC(reg) >= chip->ecc.steps,
 813					10, MTK_TIMEOUT);
 814	if (ret)
 815		dev_err(dev, "hwecc write timeout\n");
 816
 817timeout:
 818
 819	dma_unmap_single(nfc->dev, addr, len, DMA_TO_DEVICE);
 820	nfi_writel(nfc, 0, NFI_CON);
 821
 822	return ret;
 823}
 824
 825static int mtk_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 826			      const u8 *buf, int page, int raw)
 827{
 828	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 829	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 830	size_t len;
 831	const u8 *bufpoi;
 832	u32 reg;
 833	int ret;
 834
 835	mtk_nfc_select_target(chip, chip->cur_cs);
 836	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 837
 838	if (!raw) {
 839		/* OOB => FDM: from register,  ECC: from HW */
 840		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_AUTO_FMT_EN;
 841		nfi_writew(nfc, reg | CNFG_HW_ECC_EN, NFI_CNFG);
 842
 843		nfc->ecc_cfg.op = ECC_ENCODE;
 844		nfc->ecc_cfg.mode = ECC_NFI_MODE;
 845		ret = mtk_ecc_enable(nfc->ecc, &nfc->ecc_cfg);
 846		if (ret) {
 847			/* clear NFI config */
 848			reg = nfi_readw(nfc, NFI_CNFG);
 849			reg &= ~(CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
 850			nfi_writew(nfc, reg, NFI_CNFG);
 851
 852			return ret;
 853		}
 854
 855		memcpy(nfc->buffer, buf, mtd->writesize);
 856		mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, raw);
 857		bufpoi = nfc->buffer;
 858
 859		/* write OOB into the FDM registers (OOB area in MTK NAND) */
 860		mtk_nfc_write_fdm(chip);
 861	} else {
 862		bufpoi = buf;
 863	}
 864
 865	len = mtd->writesize + (raw ? mtd->oobsize : 0);
 866	ret = mtk_nfc_do_write_page(mtd, chip, bufpoi, page, len);
 867
 868	if (!raw)
 869		mtk_ecc_disable(nfc->ecc);
 870
 871	if (ret)
 872		return ret;
 873
 874	return nand_prog_page_end_op(chip);
 875}
 876
 877static int mtk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
 878				    int oob_on, int page)
 879{
 880	return mtk_nfc_write_page(nand_to_mtd(chip), chip, buf, page, 0);
 881}
 882
 883static int mtk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
 884				  int oob_on, int pg)
 885{
 886	struct mtd_info *mtd = nand_to_mtd(chip);
 887	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 888
 889	mtk_nfc_format_page(mtd, buf);
 890	return mtk_nfc_write_page(mtd, chip, nfc->buffer, pg, 1);
 891}
 892
 893static int mtk_nfc_write_subpage_hwecc(struct nand_chip *chip, u32 offset,
 894				       u32 data_len, const u8 *buf,
 895				       int oob_on, int page)
 896{
 897	struct mtd_info *mtd = nand_to_mtd(chip);
 898	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 899	int ret;
 900
 901	ret = mtk_nfc_format_subpage(mtd, offset, data_len, buf);
 902	if (ret < 0)
 903		return ret;
 904
 905	/* use the data in the private buffer (now with FDM and CRC) */
 906	return mtk_nfc_write_page(mtd, chip, nfc->buffer, page, 1);
 907}
 908
 909static int mtk_nfc_write_oob_std(struct nand_chip *chip, int page)
 910{
 911	return mtk_nfc_write_page_raw(chip, NULL, 1, page);
 912}
 913
 914static int mtk_nfc_update_ecc_stats(struct mtd_info *mtd, u8 *buf, u32 start,
 915				    u32 sectors)
 916{
 917	struct nand_chip *chip = mtd_to_nand(mtd);
 918	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 919	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 920	struct mtk_ecc_stats stats;
 921	u32 reg_size = mtk_nand->fdm.reg_size;
 922	int rc, i;
 923
 924	rc = nfi_readl(nfc, NFI_STA) & STA_EMP_PAGE;
 925	if (rc) {
 926		memset(buf, 0xff, sectors * chip->ecc.size);
 927		for (i = 0; i < sectors; i++)
 928			memset(oob_ptr(chip, start + i), 0xff, reg_size);
 929		return 0;
 930	}
 931
 932	mtk_ecc_get_stats(nfc->ecc, &stats, sectors);
 933	mtd->ecc_stats.corrected += stats.corrected;
 934	mtd->ecc_stats.failed += stats.failed;
 935
 936	return stats.bitflips;
 937}
 938
 939static int mtk_nfc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 940				u32 data_offs, u32 readlen,
 941				u8 *bufpoi, int page, int raw)
 942{
 943	struct mtk_nfc *nfc = nand_get_controller_data(chip);
 944	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
 945	u32 spare = mtk_nand->spare_per_sector;
 946	u32 column, sectors, start, end, reg;
 947	dma_addr_t addr;
 948	int bitflips = 0;
 949	size_t len;
 950	u8 *buf;
 951	int rc;
 952
 953	mtk_nfc_select_target(chip, chip->cur_cs);
 954	start = data_offs / chip->ecc.size;
 955	end = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
 956
 957	sectors = end - start;
 958	column = start * (chip->ecc.size + spare);
 959
 960	len = sectors * chip->ecc.size + (raw ? sectors * spare : 0);
 961	buf = bufpoi + start * chip->ecc.size;
 962
 963	nand_read_page_op(chip, page, column, NULL, 0);
 964
 965	addr = dma_map_single(nfc->dev, buf, len, DMA_FROM_DEVICE);
 966	rc = dma_mapping_error(nfc->dev, addr);
 967	if (rc) {
 968		dev_err(nfc->dev, "dma mapping error\n");
 969
 970		return -EINVAL;
 971	}
 972
 973	reg = nfi_readw(nfc, NFI_CNFG);
 974	reg |= CNFG_READ_EN | CNFG_DMA_BURST_EN | CNFG_AHB;
 975	if (!raw) {
 976		reg |= CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN;
 977		nfi_writew(nfc, reg, NFI_CNFG);
 978
 979		nfc->ecc_cfg.mode = ECC_NFI_MODE;
 980		nfc->ecc_cfg.sectors = sectors;
 981		nfc->ecc_cfg.op = ECC_DECODE;
 982		rc = mtk_ecc_enable(nfc->ecc, &nfc->ecc_cfg);
 983		if (rc) {
 984			dev_err(nfc->dev, "ecc enable\n");
 985			/* clear NFI_CNFG */
 986			reg &= ~(CNFG_DMA_BURST_EN | CNFG_AHB | CNFG_READ_EN |
 987				CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
 988			nfi_writew(nfc, reg, NFI_CNFG);
 989			dma_unmap_single(nfc->dev, addr, len, DMA_FROM_DEVICE);
 990
 991			return rc;
 992		}
 993	} else {
 994		nfi_writew(nfc, reg, NFI_CNFG);
 995	}
 996
 997	nfi_writel(nfc, sectors << CON_SEC_SHIFT, NFI_CON);
 998	nfi_writew(nfc, INTR_AHB_DONE_EN, NFI_INTR_EN);
 999	nfi_writel(nfc, lower_32_bits(addr), NFI_STRADDR);
1000
1001	init_completion(&nfc->done);
1002	reg = nfi_readl(nfc, NFI_CON) | CON_BRD;
1003	nfi_writel(nfc, reg, NFI_CON);
1004	nfi_writew(nfc, STAR_EN, NFI_STRDATA);
1005
1006	rc = wait_for_completion_timeout(&nfc->done, msecs_to_jiffies(500));
1007	if (!rc)
1008		dev_warn(nfc->dev, "read ahb/dma done timeout\n");
1009
1010	rc = readl_poll_timeout_atomic(nfc->regs + NFI_BYTELEN, reg,
1011				       ADDRCNTR_SEC(reg) >= sectors, 10,
1012				       MTK_TIMEOUT);
1013	if (rc < 0) {
1014		dev_err(nfc->dev, "subpage done timeout\n");
1015		bitflips = -EIO;
1016	} else if (!raw) {
1017		rc = mtk_ecc_wait_done(nfc->ecc, ECC_DECODE);
1018		bitflips = rc < 0 ? -ETIMEDOUT :
1019			mtk_nfc_update_ecc_stats(mtd, buf, start, sectors);
1020		mtk_nfc_read_fdm(chip, start, sectors);
1021	}
1022
1023	dma_unmap_single(nfc->dev, addr, len, DMA_FROM_DEVICE);
1024
1025	if (raw)
1026		goto done;
1027
1028	mtk_ecc_disable(nfc->ecc);
1029
1030	if (clamp(mtk_nand->bad_mark.sec, start, end) == mtk_nand->bad_mark.sec)
1031		mtk_nand->bad_mark.bm_swap(mtd, bufpoi, raw);
1032done:
1033	nfi_writel(nfc, 0, NFI_CON);
1034
1035	return bitflips;
1036}
1037
1038static int mtk_nfc_read_subpage_hwecc(struct nand_chip *chip, u32 off,
1039				      u32 len, u8 *p, int pg)
1040{
1041	return mtk_nfc_read_subpage(nand_to_mtd(chip), chip, off, len, p, pg,
1042				    0);
1043}
1044
1045static int mtk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *p, int oob_on,
1046				   int pg)
1047{
1048	struct mtd_info *mtd = nand_to_mtd(chip);
1049
1050	return mtk_nfc_read_subpage(mtd, chip, 0, mtd->writesize, p, pg, 0);
1051}
1052
1053static int mtk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
1054				 int page)
1055{
1056	struct mtd_info *mtd = nand_to_mtd(chip);
1057	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
1058	struct mtk_nfc *nfc = nand_get_controller_data(chip);
1059	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
1060	int i, ret;
1061
1062	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
1063	ret = mtk_nfc_read_subpage(mtd, chip, 0, mtd->writesize, nfc->buffer,
1064				   page, 1);
1065	if (ret < 0)
1066		return ret;
1067
1068	for (i = 0; i < chip->ecc.steps; i++) {
1069		memcpy(oob_ptr(chip, i), mtk_oob_ptr(chip, i), fdm->reg_size);
1070
1071		if (i == mtk_nand->bad_mark.sec)
1072			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
1073
1074		if (buf)
1075			memcpy(data_ptr(chip, buf, i), mtk_data_ptr(chip, i),
1076			       chip->ecc.size);
1077	}
1078
1079	return ret;
1080}
1081
1082static int mtk_nfc_read_oob_std(struct nand_chip *chip, int page)
1083{
1084	return mtk_nfc_read_page_raw(chip, NULL, 1, page);
1085}
1086
1087static inline void mtk_nfc_hw_init(struct mtk_nfc *nfc)
1088{
1089	/*
1090	 * CNRNB: nand ready/busy register
1091	 * -------------------------------
1092	 * 7:4: timeout register for polling the NAND busy/ready signal
1093	 * 0  : poll the status of the busy/ready signal after [7:4]*16 cycles.
1094	 */
1095	nfi_writew(nfc, 0xf1, NFI_CNRNB);
1096	nfi_writel(nfc, PAGEFMT_8K_16K, NFI_PAGEFMT);
1097
1098	mtk_nfc_hw_reset(nfc);
1099
1100	nfi_readl(nfc, NFI_INTR_STA);
1101	nfi_writel(nfc, 0, NFI_INTR_EN);
1102}
1103
1104static irqreturn_t mtk_nfc_irq(int irq, void *id)
1105{
1106	struct mtk_nfc *nfc = id;
1107	u16 sta, ien;
1108
1109	sta = nfi_readw(nfc, NFI_INTR_STA);
1110	ien = nfi_readw(nfc, NFI_INTR_EN);
1111
1112	if (!(sta & ien))
1113		return IRQ_NONE;
1114
1115	nfi_writew(nfc, ~sta & ien, NFI_INTR_EN);
1116	complete(&nfc->done);
1117
1118	return IRQ_HANDLED;
1119}
1120
1121static int mtk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
1122				  struct mtd_oob_region *oob_region)
1123{
1124	struct nand_chip *chip = mtd_to_nand(mtd);
1125	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
1126	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
1127	u32 eccsteps;
1128
1129	eccsteps = mtd->writesize / chip->ecc.size;
1130
1131	if (section >= eccsteps)
1132		return -ERANGE;
1133
1134	oob_region->length = fdm->reg_size - fdm->ecc_size;
1135	oob_region->offset = section * fdm->reg_size + fdm->ecc_size;
1136
1137	return 0;
1138}
1139
1140static int mtk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
1141				 struct mtd_oob_region *oob_region)
1142{
1143	struct nand_chip *chip = mtd_to_nand(mtd);
1144	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
1145	u32 eccsteps;
1146
1147	if (section)
1148		return -ERANGE;
1149
1150	eccsteps = mtd->writesize / chip->ecc.size;
1151	oob_region->offset = mtk_nand->fdm.reg_size * eccsteps;
1152	oob_region->length = mtd->oobsize - oob_region->offset;
1153
1154	return 0;
1155}
1156
1157static const struct mtd_ooblayout_ops mtk_nfc_ooblayout_ops = {
1158	.free = mtk_nfc_ooblayout_free,
1159	.ecc = mtk_nfc_ooblayout_ecc,
1160};
1161
1162static void mtk_nfc_set_fdm(struct mtk_nfc_fdm *fdm, struct mtd_info *mtd)
1163{
1164	struct nand_chip *nand = mtd_to_nand(mtd);
1165	struct mtk_nfc_nand_chip *chip = to_mtk_nand(nand);
1166	struct mtk_nfc *nfc = nand_get_controller_data(nand);
1167	u32 ecc_bytes;
1168
1169	ecc_bytes = DIV_ROUND_UP(nand->ecc.strength *
1170				 mtk_ecc_get_parity_bits(nfc->ecc), 8);
1171
1172	fdm->reg_size = chip->spare_per_sector - ecc_bytes;
1173	if (fdm->reg_size > NFI_FDM_MAX_SIZE)
1174		fdm->reg_size = NFI_FDM_MAX_SIZE;
1175
1176	/* bad block mark storage */
1177	fdm->ecc_size = 1;
1178}
1179
1180static void mtk_nfc_set_bad_mark_ctl(struct mtk_nfc_bad_mark_ctl *bm_ctl,
1181				     struct mtd_info *mtd)
1182{
1183	struct nand_chip *nand = mtd_to_nand(mtd);
1184
1185	if (mtd->writesize == 512) {
1186		bm_ctl->bm_swap = mtk_nfc_no_bad_mark_swap;
1187	} else {
1188		bm_ctl->bm_swap = mtk_nfc_bad_mark_swap;
1189		bm_ctl->sec = mtd->writesize / mtk_data_len(nand);
1190		bm_ctl->pos = mtd->writesize % mtk_data_len(nand);
1191	}
1192}
1193
1194static int mtk_nfc_set_spare_per_sector(u32 *sps, struct mtd_info *mtd)
1195{
1196	struct nand_chip *nand = mtd_to_nand(mtd);
1197	struct mtk_nfc *nfc = nand_get_controller_data(nand);
1198	const u8 *spare = nfc->caps->spare_size;
1199	u32 eccsteps, i, closest_spare = 0;
1200
1201	eccsteps = mtd->writesize / nand->ecc.size;
1202	*sps = mtd->oobsize / eccsteps;
1203
1204	if (nand->ecc.size == 1024)
1205		*sps >>= 1;
1206
1207	if (*sps < MTK_NFC_MIN_SPARE)
1208		return -EINVAL;
1209
1210	for (i = 0; i < nfc->caps->num_spare_size; i++) {
1211		if (*sps >= spare[i] && spare[i] >= spare[closest_spare]) {
1212			closest_spare = i;
1213			if (*sps == spare[i])
1214				break;
1215		}
1216	}
1217
1218	*sps = spare[closest_spare];
1219
1220	if (nand->ecc.size == 1024)
1221		*sps <<= 1;
1222
1223	return 0;
1224}
1225
1226static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
1227{
1228	struct nand_chip *nand = mtd_to_nand(mtd);
1229	const struct nand_ecc_props *requirements =
1230		nanddev_get_ecc_requirements(&nand->base);
1231	struct mtk_nfc *nfc = nand_get_controller_data(nand);
1232	u32 spare;
1233	int free, ret;
1234
1235	/* support only ecc hw mode */
1236	if (nand->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) {
1237		dev_err(dev, "ecc.engine_type not supported\n");
1238		return -EINVAL;
1239	}
1240
1241	/* if optional dt settings not present */
1242	if (!nand->ecc.size || !nand->ecc.strength) {
1243		/* use datasheet requirements */
1244		nand->ecc.strength = requirements->strength;
1245		nand->ecc.size = requirements->step_size;
1246
1247		/*
1248		 * align eccstrength and eccsize
1249		 * this controller only supports 512 and 1024 sizes
1250		 */
1251		if (nand->ecc.size < 1024) {
1252			if (mtd->writesize > 512 &&
1253			    nfc->caps->max_sector_size > 512) {
1254				nand->ecc.size = 1024;
1255				nand->ecc.strength <<= 1;
1256			} else {
1257				nand->ecc.size = 512;
1258			}
1259		} else {
1260			nand->ecc.size = 1024;
1261		}
1262
1263		ret = mtk_nfc_set_spare_per_sector(&spare, mtd);
1264		if (ret)
1265			return ret;
1266
1267		/* calculate oob bytes except ecc parity data */
1268		free = (nand->ecc.strength * mtk_ecc_get_parity_bits(nfc->ecc)
1269			+ 7) >> 3;
1270		free = spare - free;
1271
1272		/*
1273		 * enhance ecc strength if oob left is bigger than max FDM size
1274		 * or reduce ecc strength if oob size is not enough for ecc
1275		 * parity data.
1276		 */
1277		if (free > NFI_FDM_MAX_SIZE) {
1278			spare -= NFI_FDM_MAX_SIZE;
1279			nand->ecc.strength = (spare << 3) /
1280					     mtk_ecc_get_parity_bits(nfc->ecc);
1281		} else if (free < 0) {
1282			spare -= NFI_FDM_MIN_SIZE;
1283			nand->ecc.strength = (spare << 3) /
1284					     mtk_ecc_get_parity_bits(nfc->ecc);
1285		}
1286	}
1287
1288	mtk_ecc_adjust_strength(nfc->ecc, &nand->ecc.strength);
1289
1290	dev_info(dev, "eccsize %d eccstrength %d\n",
1291		 nand->ecc.size, nand->ecc.strength);
1292
1293	return 0;
1294}
1295
1296static int mtk_nfc_attach_chip(struct nand_chip *chip)
1297{
1298	struct mtd_info *mtd = nand_to_mtd(chip);
1299	struct device *dev = mtd->dev.parent;
1300	struct mtk_nfc *nfc = nand_get_controller_data(chip);
1301	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
1302	int len;
1303	int ret;
1304
1305	if (chip->options & NAND_BUSWIDTH_16) {
1306		dev_err(dev, "16bits buswidth not supported");
1307		return -EINVAL;
1308	}
1309
1310	/* store bbt magic in page, cause OOB is not protected */
1311	if (chip->bbt_options & NAND_BBT_USE_FLASH)
1312		chip->bbt_options |= NAND_BBT_NO_OOB;
1313
1314	ret = mtk_nfc_ecc_init(dev, mtd);
1315	if (ret)
1316		return ret;
1317
1318	ret = mtk_nfc_set_spare_per_sector(&mtk_nand->spare_per_sector, mtd);
1319	if (ret)
1320		return ret;
1321
1322	mtk_nfc_set_fdm(&mtk_nand->fdm, mtd);
1323	mtk_nfc_set_bad_mark_ctl(&mtk_nand->bad_mark, mtd);
1324
1325	len = mtd->writesize + mtd->oobsize;
1326	nfc->buffer = devm_kzalloc(dev, len, GFP_KERNEL);
1327	if (!nfc->buffer)
1328		return  -ENOMEM;
1329
1330	return 0;
1331}
1332
1333static const struct nand_controller_ops mtk_nfc_controller_ops = {
1334	.attach_chip = mtk_nfc_attach_chip,
1335	.setup_interface = mtk_nfc_setup_interface,
1336	.exec_op = mtk_nfc_exec_op,
1337};
1338
1339static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc,
1340				  struct device_node *np)
1341{
1342	struct mtk_nfc_nand_chip *chip;
1343	struct nand_chip *nand;
1344	struct mtd_info *mtd;
1345	int nsels;
1346	u32 tmp;
1347	int ret;
1348	int i;
1349
1350	if (!of_get_property(np, "reg", &nsels))
1351		return -ENODEV;
1352
1353	nsels /= sizeof(u32);
1354	if (!nsels || nsels > MTK_NAND_MAX_NSELS) {
1355		dev_err(dev, "invalid reg property size %d\n", nsels);
1356		return -EINVAL;
1357	}
1358
1359	chip = devm_kzalloc(dev, sizeof(*chip) + nsels * sizeof(u8),
1360			    GFP_KERNEL);
1361	if (!chip)
1362		return -ENOMEM;
1363
1364	chip->nsels = nsels;
1365	for (i = 0; i < nsels; i++) {
1366		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1367		if (ret) {
1368			dev_err(dev, "reg property failure : %d\n", ret);
1369			return ret;
1370		}
1371
1372		if (tmp >= MTK_NAND_MAX_NSELS) {
1373			dev_err(dev, "invalid CS: %u\n", tmp);
1374			return -EINVAL;
1375		}
1376
1377		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1378			dev_err(dev, "CS %u already assigned\n", tmp);
1379			return -EINVAL;
1380		}
1381
1382		chip->sels[i] = tmp;
1383	}
1384
1385	nand = &chip->nand;
1386	nand->controller = &nfc->controller;
1387
1388	nand_set_flash_node(nand, np);
1389	nand_set_controller_data(nand, nfc);
1390
1391	nand->options |= NAND_USES_DMA | NAND_SUBPAGE_READ;
1392
1393	/* set default mode in case dt entry is missing */
1394	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1395
1396	nand->ecc.write_subpage = mtk_nfc_write_subpage_hwecc;
1397	nand->ecc.write_page_raw = mtk_nfc_write_page_raw;
1398	nand->ecc.write_page = mtk_nfc_write_page_hwecc;
1399	nand->ecc.write_oob_raw = mtk_nfc_write_oob_std;
1400	nand->ecc.write_oob = mtk_nfc_write_oob_std;
1401
1402	nand->ecc.read_subpage = mtk_nfc_read_subpage_hwecc;
1403	nand->ecc.read_page_raw = mtk_nfc_read_page_raw;
1404	nand->ecc.read_page = mtk_nfc_read_page_hwecc;
1405	nand->ecc.read_oob_raw = mtk_nfc_read_oob_std;
1406	nand->ecc.read_oob = mtk_nfc_read_oob_std;
1407
1408	mtd = nand_to_mtd(nand);
1409	mtd->owner = THIS_MODULE;
1410	mtd->dev.parent = dev;
1411	mtd->name = MTK_NAME;
1412	mtd_set_ooblayout(mtd, &mtk_nfc_ooblayout_ops);
1413
1414	mtk_nfc_hw_init(nfc);
1415
1416	ret = nand_scan(nand, nsels);
1417	if (ret)
1418		return ret;
1419
1420	ret = mtd_device_register(mtd, NULL, 0);
1421	if (ret) {
1422		dev_err(dev, "mtd parse partition error\n");
1423		nand_cleanup(nand);
1424		return ret;
1425	}
1426
1427	list_add_tail(&chip->node, &nfc->chips);
1428
1429	return 0;
1430}
1431
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1432static int mtk_nfc_nand_chips_init(struct device *dev, struct mtk_nfc *nfc)
1433{
1434	struct device_node *np = dev->of_node;
1435	struct device_node *nand_np;
1436	int ret;
1437
1438	for_each_child_of_node(np, nand_np) {
1439		ret = mtk_nfc_nand_chip_init(dev, nfc, nand_np);
1440		if (ret) {
1441			of_node_put(nand_np);
1442			return ret;
1443		}
1444	}
1445
1446	return 0;
1447}
1448
1449static const struct mtk_nfc_caps mtk_nfc_caps_mt2701 = {
1450	.spare_size = spare_size_mt2701,
1451	.num_spare_size = 16,
1452	.pageformat_spare_shift = 4,
1453	.nfi_clk_div = 1,
1454	.max_sector = 16,
1455	.max_sector_size = 1024,
1456};
1457
1458static const struct mtk_nfc_caps mtk_nfc_caps_mt2712 = {
1459	.spare_size = spare_size_mt2712,
1460	.num_spare_size = 19,
1461	.pageformat_spare_shift = 16,
1462	.nfi_clk_div = 2,
1463	.max_sector = 16,
1464	.max_sector_size = 1024,
1465};
1466
1467static const struct mtk_nfc_caps mtk_nfc_caps_mt7622 = {
1468	.spare_size = spare_size_mt7622,
1469	.num_spare_size = 4,
1470	.pageformat_spare_shift = 4,
1471	.nfi_clk_div = 1,
1472	.max_sector = 8,
1473	.max_sector_size = 512,
1474};
1475
1476static const struct of_device_id mtk_nfc_id_table[] = {
1477	{
1478		.compatible = "mediatek,mt2701-nfc",
1479		.data = &mtk_nfc_caps_mt2701,
1480	}, {
1481		.compatible = "mediatek,mt2712-nfc",
1482		.data = &mtk_nfc_caps_mt2712,
1483	}, {
1484		.compatible = "mediatek,mt7622-nfc",
1485		.data = &mtk_nfc_caps_mt7622,
1486	},
1487	{}
1488};
1489MODULE_DEVICE_TABLE(of, mtk_nfc_id_table);
1490
1491static int mtk_nfc_probe(struct platform_device *pdev)
1492{
1493	struct device *dev = &pdev->dev;
1494	struct device_node *np = dev->of_node;
1495	struct mtk_nfc *nfc;
1496	int ret, irq;
1497
1498	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1499	if (!nfc)
1500		return -ENOMEM;
1501
1502	nand_controller_init(&nfc->controller);
1503	INIT_LIST_HEAD(&nfc->chips);
1504	nfc->controller.ops = &mtk_nfc_controller_ops;
1505
1506	/* probe defer if not ready */
1507	nfc->ecc = of_mtk_ecc_get(np);
1508	if (IS_ERR(nfc->ecc))
1509		return PTR_ERR(nfc->ecc);
1510	else if (!nfc->ecc)
1511		return -ENODEV;
1512
1513	nfc->caps = of_device_get_match_data(dev);
1514	nfc->dev = dev;
1515
1516	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
1517	if (IS_ERR(nfc->regs)) {
1518		ret = PTR_ERR(nfc->regs);
1519		goto release_ecc;
1520	}
1521
1522	nfc->clk.nfi_clk = devm_clk_get_enabled(dev, "nfi_clk");
1523	if (IS_ERR(nfc->clk.nfi_clk)) {
1524		dev_err(dev, "no clk\n");
1525		ret = PTR_ERR(nfc->clk.nfi_clk);
1526		goto release_ecc;
1527	}
1528
1529	nfc->clk.pad_clk = devm_clk_get_enabled(dev, "pad_clk");
1530	if (IS_ERR(nfc->clk.pad_clk)) {
1531		dev_err(dev, "no pad clk\n");
1532		ret = PTR_ERR(nfc->clk.pad_clk);
1533		goto release_ecc;
1534	}
1535
1536	irq = platform_get_irq(pdev, 0);
1537	if (irq < 0) {
1538		ret = -EINVAL;
1539		goto release_ecc;
1540	}
1541
1542	ret = devm_request_irq(dev, irq, mtk_nfc_irq, 0x0, "mtk-nand", nfc);
1543	if (ret) {
1544		dev_err(dev, "failed to request nfi irq\n");
1545		goto release_ecc;
1546	}
1547
1548	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1549	if (ret) {
1550		dev_err(dev, "failed to set dma mask\n");
1551		goto release_ecc;
1552	}
1553
1554	platform_set_drvdata(pdev, nfc);
1555
1556	ret = mtk_nfc_nand_chips_init(dev, nfc);
1557	if (ret) {
1558		dev_err(dev, "failed to init nand chips\n");
1559		goto release_ecc;
1560	}
1561
1562	return 0;
1563
1564release_ecc:
1565	mtk_ecc_release(nfc->ecc);
1566
1567	return ret;
1568}
1569
1570static void mtk_nfc_remove(struct platform_device *pdev)
1571{
1572	struct mtk_nfc *nfc = platform_get_drvdata(pdev);
1573	struct mtk_nfc_nand_chip *mtk_chip;
1574	struct nand_chip *chip;
1575	int ret;
1576
1577	while (!list_empty(&nfc->chips)) {
1578		mtk_chip = list_first_entry(&nfc->chips,
1579					    struct mtk_nfc_nand_chip, node);
1580		chip = &mtk_chip->nand;
1581		ret = mtd_device_unregister(nand_to_mtd(chip));
1582		WARN_ON(ret);
1583		nand_cleanup(chip);
1584		list_del(&mtk_chip->node);
1585	}
1586
 
1587	mtk_ecc_release(nfc->ecc);
1588}
1589
1590#ifdef CONFIG_PM_SLEEP
1591static int mtk_nfc_suspend(struct device *dev)
1592{
1593	struct mtk_nfc *nfc = dev_get_drvdata(dev);
1594
1595	clk_disable_unprepare(nfc->clk.nfi_clk);
1596	clk_disable_unprepare(nfc->clk.pad_clk);
1597
1598	return 0;
1599}
1600
1601static int mtk_nfc_resume(struct device *dev)
1602{
1603	struct mtk_nfc *nfc = dev_get_drvdata(dev);
1604	struct mtk_nfc_nand_chip *chip;
1605	struct nand_chip *nand;
1606	int ret;
1607	u32 i;
1608
1609	udelay(200);
1610
1611	ret = clk_prepare_enable(nfc->clk.nfi_clk);
1612	if (ret) {
1613		dev_err(dev, "failed to enable nfi clk\n");
1614		return ret;
1615	}
1616
1617	ret = clk_prepare_enable(nfc->clk.pad_clk);
1618	if (ret) {
1619		dev_err(dev, "failed to enable pad clk\n");
1620		clk_disable_unprepare(nfc->clk.nfi_clk);
1621		return ret;
1622	}
1623
1624	/* reset NAND chip if VCC was powered off */
1625	list_for_each_entry(chip, &nfc->chips, node) {
1626		nand = &chip->nand;
1627		for (i = 0; i < chip->nsels; i++)
1628			nand_reset(nand, i);
1629	}
1630
1631	return 0;
1632}
1633
1634static SIMPLE_DEV_PM_OPS(mtk_nfc_pm_ops, mtk_nfc_suspend, mtk_nfc_resume);
1635#endif
1636
1637static struct platform_driver mtk_nfc_driver = {
1638	.probe  = mtk_nfc_probe,
1639	.remove_new = mtk_nfc_remove,
1640	.driver = {
1641		.name  = MTK_NAME,
1642		.of_match_table = mtk_nfc_id_table,
1643#ifdef CONFIG_PM_SLEEP
1644		.pm = &mtk_nfc_pm_ops,
1645#endif
1646	},
1647};
1648
1649module_platform_driver(mtk_nfc_driver);
1650
1651MODULE_LICENSE("Dual MIT/GPL");
1652MODULE_AUTHOR("Xiaolei Li <xiaolei.li@mediatek.com>");
1653MODULE_DESCRIPTION("MTK Nand Flash Controller Driver");