Loading...
Note: File does not exist in v6.8.
1/*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25#include <linux/dma-mapping.h>
26#include <linux/slab.h>
27#include <linux/module.h>
28#include <linux/moduleparam.h>
29#include <linux/platform_device.h>
30#include <linux/of.h>
31#include <linux/of_device.h>
32#include <linux/of_gpio.h>
33#include <linux/mtd/mtd.h>
34#include <linux/mtd/nand.h>
35#include <linux/mtd/partitions.h>
36#include <linux/clk.h>
37#include <linux/delay.h>
38#include <linux/dmaengine.h>
39#include <linux/gpio.h>
40#include <linux/interrupt.h>
41#include <linux/iopoll.h>
42#include <linux/reset.h>
43
44#define NFC_REG_CTL 0x0000
45#define NFC_REG_ST 0x0004
46#define NFC_REG_INT 0x0008
47#define NFC_REG_TIMING_CTL 0x000C
48#define NFC_REG_TIMING_CFG 0x0010
49#define NFC_REG_ADDR_LOW 0x0014
50#define NFC_REG_ADDR_HIGH 0x0018
51#define NFC_REG_SECTOR_NUM 0x001C
52#define NFC_REG_CNT 0x0020
53#define NFC_REG_CMD 0x0024
54#define NFC_REG_RCMD_SET 0x0028
55#define NFC_REG_WCMD_SET 0x002C
56#define NFC_REG_IO_DATA 0x0030
57#define NFC_REG_ECC_CTL 0x0034
58#define NFC_REG_ECC_ST 0x0038
59#define NFC_REG_DEBUG 0x003C
60#define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
61#define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
62#define NFC_REG_SPARE_AREA 0x00A0
63#define NFC_REG_PAT_ID 0x00A4
64#define NFC_RAM0_BASE 0x0400
65#define NFC_RAM1_BASE 0x0800
66
67/* define bit use in NFC_CTL */
68#define NFC_EN BIT(0)
69#define NFC_RESET BIT(1)
70#define NFC_BUS_WIDTH_MSK BIT(2)
71#define NFC_BUS_WIDTH_8 (0 << 2)
72#define NFC_BUS_WIDTH_16 (1 << 2)
73#define NFC_RB_SEL_MSK BIT(3)
74#define NFC_RB_SEL(x) ((x) << 3)
75#define NFC_CE_SEL_MSK GENMASK(26, 24)
76#define NFC_CE_SEL(x) ((x) << 24)
77#define NFC_CE_CTL BIT(6)
78#define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
79#define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
80#define NFC_SAM BIT(12)
81#define NFC_RAM_METHOD BIT(14)
82#define NFC_DEBUG_CTL BIT(31)
83
84/* define bit use in NFC_ST */
85#define NFC_RB_B2R BIT(0)
86#define NFC_CMD_INT_FLAG BIT(1)
87#define NFC_DMA_INT_FLAG BIT(2)
88#define NFC_CMD_FIFO_STATUS BIT(3)
89#define NFC_STA BIT(4)
90#define NFC_NATCH_INT_FLAG BIT(5)
91#define NFC_RB_STATE(x) BIT(x + 8)
92
93/* define bit use in NFC_INT */
94#define NFC_B2R_INT_ENABLE BIT(0)
95#define NFC_CMD_INT_ENABLE BIT(1)
96#define NFC_DMA_INT_ENABLE BIT(2)
97#define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
98 NFC_CMD_INT_ENABLE | \
99 NFC_DMA_INT_ENABLE)
100
101/* define bit use in NFC_TIMING_CTL */
102#define NFC_TIMING_CTL_EDO BIT(8)
103
104/* define NFC_TIMING_CFG register layout */
105#define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
108 (((tCAD) & 0x7) << 8))
109
110/* define bit use in NFC_CMD */
111#define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
112#define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
113#define NFC_CMD(x) (x)
114#define NFC_ADR_NUM_MSK GENMASK(18, 16)
115#define NFC_ADR_NUM(x) (((x) - 1) << 16)
116#define NFC_SEND_ADR BIT(19)
117#define NFC_ACCESS_DIR BIT(20)
118#define NFC_DATA_TRANS BIT(21)
119#define NFC_SEND_CMD1 BIT(22)
120#define NFC_WAIT_FLAG BIT(23)
121#define NFC_SEND_CMD2 BIT(24)
122#define NFC_SEQ BIT(25)
123#define NFC_DATA_SWAP_METHOD BIT(26)
124#define NFC_ROW_AUTO_INC BIT(27)
125#define NFC_SEND_CMD3 BIT(28)
126#define NFC_SEND_CMD4 BIT(29)
127#define NFC_CMD_TYPE_MSK GENMASK(31, 30)
128#define NFC_NORMAL_OP (0 << 30)
129#define NFC_ECC_OP (1 << 30)
130#define NFC_PAGE_OP (2 << 30)
131
132/* define bit use in NFC_RCMD_SET */
133#define NFC_READ_CMD_MSK GENMASK(7, 0)
134#define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
135#define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
136
137/* define bit use in NFC_WCMD_SET */
138#define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
139#define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
140#define NFC_READ_CMD0_MSK GENMASK(23, 16)
141#define NFC_READ_CMD1_MSK GENMASK(31, 24)
142
143/* define bit use in NFC_ECC_CTL */
144#define NFC_ECC_EN BIT(0)
145#define NFC_ECC_PIPELINE BIT(3)
146#define NFC_ECC_EXCEPTION BIT(4)
147#define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
148#define NFC_ECC_BLOCK_512 BIT(5)
149#define NFC_RANDOM_EN BIT(9)
150#define NFC_RANDOM_DIRECTION BIT(10)
151#define NFC_ECC_MODE_MSK GENMASK(15, 12)
152#define NFC_ECC_MODE(x) ((x) << 12)
153#define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
154#define NFC_RANDOM_SEED(x) ((x) << 16)
155
156/* define bit use in NFC_ECC_ST */
157#define NFC_ECC_ERR(x) BIT(x)
158#define NFC_ECC_ERR_MSK GENMASK(15, 0)
159#define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
160#define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
161
162#define NFC_DEFAULT_TIMEOUT_MS 1000
163
164#define NFC_SRAM_SIZE 1024
165
166#define NFC_MAX_CS 7
167
168/*
169 * Ready/Busy detection type: describes the Ready/Busy detection modes
170 *
171 * @RB_NONE: no external detection available, rely on STATUS command
172 * and software timeouts
173 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
174 * pin of the NAND flash chip must be connected to one of the
175 * native NAND R/B pins (those which can be muxed to the NAND
176 * Controller)
177 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
178 * pin of the NAND flash chip must be connected to a GPIO capable
179 * pin.
180 */
181enum sunxi_nand_rb_type {
182 RB_NONE,
183 RB_NATIVE,
184 RB_GPIO,
185};
186
187/*
188 * Ready/Busy structure: stores information related to Ready/Busy detection
189 *
190 * @type: the Ready/Busy detection mode
191 * @info: information related to the R/B detection mode. Either a gpio
192 * id or a native R/B id (those supported by the NAND controller).
193 */
194struct sunxi_nand_rb {
195 enum sunxi_nand_rb_type type;
196 union {
197 int gpio;
198 int nativeid;
199 } info;
200};
201
202/*
203 * Chip Select structure: stores information related to NAND Chip Select
204 *
205 * @cs: the NAND CS id used to communicate with a NAND Chip
206 * @rb: the Ready/Busy description
207 */
208struct sunxi_nand_chip_sel {
209 u8 cs;
210 struct sunxi_nand_rb rb;
211};
212
213/*
214 * sunxi HW ECC infos: stores information related to HW ECC support
215 *
216 * @mode: the sunxi ECC mode field deduced from ECC requirements
217 */
218struct sunxi_nand_hw_ecc {
219 int mode;
220};
221
222/*
223 * NAND chip structure: stores NAND chip device related information
224 *
225 * @node: used to store NAND chips into a list
226 * @nand: base NAND chip structure
227 * @mtd: base MTD structure
228 * @clk_rate: clk_rate required for this NAND chip
229 * @timing_cfg TIMING_CFG register value for this NAND chip
230 * @selected: current active CS
231 * @nsels: number of CS lines required by the NAND chip
232 * @sels: array of CS lines descriptions
233 */
234struct sunxi_nand_chip {
235 struct list_head node;
236 struct nand_chip nand;
237 unsigned long clk_rate;
238 u32 timing_cfg;
239 u32 timing_ctl;
240 int selected;
241 int addr_cycles;
242 u32 addr[2];
243 int cmd_cycles;
244 u8 cmd[2];
245 int nsels;
246 struct sunxi_nand_chip_sel sels[0];
247};
248
249static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
250{
251 return container_of(nand, struct sunxi_nand_chip, nand);
252}
253
254/*
255 * NAND Controller structure: stores sunxi NAND controller information
256 *
257 * @controller: base controller structure
258 * @dev: parent device (used to print error messages)
259 * @regs: NAND controller registers
260 * @ahb_clk: NAND Controller AHB clock
261 * @mod_clk: NAND Controller mod clock
262 * @assigned_cs: bitmask describing already assigned CS lines
263 * @clk_rate: NAND controller current clock rate
264 * @chips: a list containing all the NAND chips attached to
265 * this NAND controller
266 * @complete: a completion object used to wait for NAND
267 * controller events
268 */
269struct sunxi_nfc {
270 struct nand_hw_control controller;
271 struct device *dev;
272 void __iomem *regs;
273 struct clk *ahb_clk;
274 struct clk *mod_clk;
275 struct reset_control *reset;
276 unsigned long assigned_cs;
277 unsigned long clk_rate;
278 struct list_head chips;
279 struct completion complete;
280 struct dma_chan *dmac;
281};
282
283static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
284{
285 return container_of(ctrl, struct sunxi_nfc, controller);
286}
287
288static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
289{
290 struct sunxi_nfc *nfc = dev_id;
291 u32 st = readl(nfc->regs + NFC_REG_ST);
292 u32 ien = readl(nfc->regs + NFC_REG_INT);
293
294 if (!(ien & st))
295 return IRQ_NONE;
296
297 if ((ien & st) == ien)
298 complete(&nfc->complete);
299
300 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
301 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
302
303 return IRQ_HANDLED;
304}
305
306static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
307 bool use_polling, unsigned int timeout_ms)
308{
309 int ret;
310
311 if (events & ~NFC_INT_MASK)
312 return -EINVAL;
313
314 if (!timeout_ms)
315 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
316
317 if (!use_polling) {
318 init_completion(&nfc->complete);
319
320 writel(events, nfc->regs + NFC_REG_INT);
321
322 ret = wait_for_completion_timeout(&nfc->complete,
323 msecs_to_jiffies(timeout_ms));
324
325 writel(0, nfc->regs + NFC_REG_INT);
326 } else {
327 u32 status;
328
329 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
330 (status & events) == events, 1,
331 timeout_ms * 1000);
332 }
333
334 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
335
336 if (ret)
337 dev_err(nfc->dev, "wait interrupt timedout\n");
338
339 return ret;
340}
341
342static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
343{
344 u32 status;
345 int ret;
346
347 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
348 !(status & NFC_CMD_FIFO_STATUS), 1,
349 NFC_DEFAULT_TIMEOUT_MS * 1000);
350 if (ret)
351 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
352
353 return ret;
354}
355
356static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
357{
358 u32 ctl;
359 int ret;
360
361 writel(0, nfc->regs + NFC_REG_ECC_CTL);
362 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
363
364 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
365 !(ctl & NFC_RESET), 1,
366 NFC_DEFAULT_TIMEOUT_MS * 1000);
367 if (ret)
368 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
369
370 return ret;
371}
372
373static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
374 int chunksize, int nchunks,
375 enum dma_data_direction ddir,
376 struct scatterlist *sg)
377{
378 struct nand_chip *nand = mtd_to_nand(mtd);
379 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
380 struct dma_async_tx_descriptor *dmad;
381 enum dma_transfer_direction tdir;
382 dma_cookie_t dmat;
383 int ret;
384
385 if (ddir == DMA_FROM_DEVICE)
386 tdir = DMA_DEV_TO_MEM;
387 else
388 tdir = DMA_MEM_TO_DEV;
389
390 sg_init_one(sg, buf, nchunks * chunksize);
391 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
392 if (!ret)
393 return -ENOMEM;
394
395 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
396 if (!dmad) {
397 ret = -EINVAL;
398 goto err_unmap_buf;
399 }
400
401 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
402 nfc->regs + NFC_REG_CTL);
403 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
404 writel(chunksize, nfc->regs + NFC_REG_CNT);
405 dmat = dmaengine_submit(dmad);
406
407 ret = dma_submit_error(dmat);
408 if (ret)
409 goto err_clr_dma_flag;
410
411 return 0;
412
413err_clr_dma_flag:
414 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
415 nfc->regs + NFC_REG_CTL);
416
417err_unmap_buf:
418 dma_unmap_sg(nfc->dev, sg, 1, ddir);
419 return ret;
420}
421
422static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
423 enum dma_data_direction ddir,
424 struct scatterlist *sg)
425{
426 struct nand_chip *nand = mtd_to_nand(mtd);
427 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
428
429 dma_unmap_sg(nfc->dev, sg, 1, ddir);
430 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
431 nfc->regs + NFC_REG_CTL);
432}
433
434static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
435{
436 struct nand_chip *nand = mtd_to_nand(mtd);
437 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
438 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
439 struct sunxi_nand_rb *rb;
440 int ret;
441
442 if (sunxi_nand->selected < 0)
443 return 0;
444
445 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
446
447 switch (rb->type) {
448 case RB_NATIVE:
449 ret = !!(readl(nfc->regs + NFC_REG_ST) &
450 NFC_RB_STATE(rb->info.nativeid));
451 break;
452 case RB_GPIO:
453 ret = gpio_get_value(rb->info.gpio);
454 break;
455 case RB_NONE:
456 default:
457 ret = 0;
458 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
459 break;
460 }
461
462 return ret;
463}
464
465static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
466{
467 struct nand_chip *nand = mtd_to_nand(mtd);
468 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
469 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
470 struct sunxi_nand_chip_sel *sel;
471 u32 ctl;
472
473 if (chip > 0 && chip >= sunxi_nand->nsels)
474 return;
475
476 if (chip == sunxi_nand->selected)
477 return;
478
479 ctl = readl(nfc->regs + NFC_REG_CTL) &
480 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
481
482 if (chip >= 0) {
483 sel = &sunxi_nand->sels[chip];
484
485 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
486 NFC_PAGE_SHIFT(nand->page_shift);
487 if (sel->rb.type == RB_NONE) {
488 nand->dev_ready = NULL;
489 } else {
490 nand->dev_ready = sunxi_nfc_dev_ready;
491 if (sel->rb.type == RB_NATIVE)
492 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
493 }
494
495 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
496
497 if (nfc->clk_rate != sunxi_nand->clk_rate) {
498 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
499 nfc->clk_rate = sunxi_nand->clk_rate;
500 }
501 }
502
503 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
504 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
505 writel(ctl, nfc->regs + NFC_REG_CTL);
506
507 sunxi_nand->selected = chip;
508}
509
510static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
511{
512 struct nand_chip *nand = mtd_to_nand(mtd);
513 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
514 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
515 int ret;
516 int cnt;
517 int offs = 0;
518 u32 tmp;
519
520 while (len > offs) {
521 cnt = min(len - offs, NFC_SRAM_SIZE);
522
523 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
524 if (ret)
525 break;
526
527 writel(cnt, nfc->regs + NFC_REG_CNT);
528 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
529 writel(tmp, nfc->regs + NFC_REG_CMD);
530
531 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
532 if (ret)
533 break;
534
535 if (buf)
536 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
537 cnt);
538 offs += cnt;
539 }
540}
541
542static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
543 int len)
544{
545 struct nand_chip *nand = mtd_to_nand(mtd);
546 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
547 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
548 int ret;
549 int cnt;
550 int offs = 0;
551 u32 tmp;
552
553 while (len > offs) {
554 cnt = min(len - offs, NFC_SRAM_SIZE);
555
556 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
557 if (ret)
558 break;
559
560 writel(cnt, nfc->regs + NFC_REG_CNT);
561 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
562 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
563 NFC_ACCESS_DIR;
564 writel(tmp, nfc->regs + NFC_REG_CMD);
565
566 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
567 if (ret)
568 break;
569
570 offs += cnt;
571 }
572}
573
574static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
575{
576 uint8_t ret;
577
578 sunxi_nfc_read_buf(mtd, &ret, 1);
579
580 return ret;
581}
582
583static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
584 unsigned int ctrl)
585{
586 struct nand_chip *nand = mtd_to_nand(mtd);
587 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
588 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
589 int ret;
590
591 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
592 if (ret)
593 return;
594
595 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
596 !(ctrl & (NAND_CLE | NAND_ALE))) {
597 u32 cmd = 0;
598
599 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
600 return;
601
602 if (sunxi_nand->cmd_cycles--)
603 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
604
605 if (sunxi_nand->cmd_cycles--) {
606 cmd |= NFC_SEND_CMD2;
607 writel(sunxi_nand->cmd[1],
608 nfc->regs + NFC_REG_RCMD_SET);
609 }
610
611 sunxi_nand->cmd_cycles = 0;
612
613 if (sunxi_nand->addr_cycles) {
614 cmd |= NFC_SEND_ADR |
615 NFC_ADR_NUM(sunxi_nand->addr_cycles);
616 writel(sunxi_nand->addr[0],
617 nfc->regs + NFC_REG_ADDR_LOW);
618 }
619
620 if (sunxi_nand->addr_cycles > 4)
621 writel(sunxi_nand->addr[1],
622 nfc->regs + NFC_REG_ADDR_HIGH);
623
624 writel(cmd, nfc->regs + NFC_REG_CMD);
625 sunxi_nand->addr[0] = 0;
626 sunxi_nand->addr[1] = 0;
627 sunxi_nand->addr_cycles = 0;
628 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
629 }
630
631 if (ctrl & NAND_CLE) {
632 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
633 } else if (ctrl & NAND_ALE) {
634 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
635 dat << ((sunxi_nand->addr_cycles % 4) * 8);
636 sunxi_nand->addr_cycles++;
637 }
638}
639
640/* These seed values have been extracted from Allwinner's BSP */
641static const u16 sunxi_nfc_randomizer_page_seeds[] = {
642 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
643 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
644 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
645 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
646 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
647 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
648 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
649 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
650 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
651 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
652 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
653 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
654 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
655 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
656 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
657 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
658};
659
660/*
661 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
662 * have been generated using
663 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
664 * the randomizer engine does internally before de/scrambling OOB data.
665 *
666 * Those tables are statically defined to avoid calculating randomizer state
667 * at runtime.
668 */
669static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
670 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
671 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
672 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
673 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
674 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
675 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
676 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
677 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
678 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
679 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
680 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
681 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
682 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
683 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
684 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
685 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
686};
687
688static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
689 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
690 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
691 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
692 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
693 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
694 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
695 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
696 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
697 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
698 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
699 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
700 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
701 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
702 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
703 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
704 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
705};
706
707static u16 sunxi_nfc_randomizer_step(u16 state, int count)
708{
709 state &= 0x7fff;
710
711 /*
712 * This loop is just a simple implementation of a Fibonacci LFSR using
713 * the x16 + x15 + 1 polynomial.
714 */
715 while (count--)
716 state = ((state >> 1) |
717 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
718
719 return state;
720}
721
722static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
723{
724 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
725 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
726
727 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
728 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
729
730 if (ecc) {
731 if (mtd->ecc_step_size == 512)
732 seeds = sunxi_nfc_randomizer_ecc512_seeds;
733 else
734 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
735 }
736
737 return seeds[page % mod];
738}
739
740static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
741 int page, bool ecc)
742{
743 struct nand_chip *nand = mtd_to_nand(mtd);
744 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
745 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
746 u16 state;
747
748 if (!(nand->options & NAND_NEED_SCRAMBLING))
749 return;
750
751 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
752 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
753 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
754 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
755}
756
757static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
758{
759 struct nand_chip *nand = mtd_to_nand(mtd);
760 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
761
762 if (!(nand->options & NAND_NEED_SCRAMBLING))
763 return;
764
765 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
766 nfc->regs + NFC_REG_ECC_CTL);
767}
768
769static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
770{
771 struct nand_chip *nand = mtd_to_nand(mtd);
772 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
773
774 if (!(nand->options & NAND_NEED_SCRAMBLING))
775 return;
776
777 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
778 nfc->regs + NFC_REG_ECC_CTL);
779}
780
781static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
782{
783 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
784
785 bbm[0] ^= state;
786 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
787}
788
789static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
790 const uint8_t *buf, int len,
791 bool ecc, int page)
792{
793 sunxi_nfc_randomizer_config(mtd, page, ecc);
794 sunxi_nfc_randomizer_enable(mtd);
795 sunxi_nfc_write_buf(mtd, buf, len);
796 sunxi_nfc_randomizer_disable(mtd);
797}
798
799static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
800 int len, bool ecc, int page)
801{
802 sunxi_nfc_randomizer_config(mtd, page, ecc);
803 sunxi_nfc_randomizer_enable(mtd);
804 sunxi_nfc_read_buf(mtd, buf, len);
805 sunxi_nfc_randomizer_disable(mtd);
806}
807
808static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
809{
810 struct nand_chip *nand = mtd_to_nand(mtd);
811 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
812 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
813 u32 ecc_ctl;
814
815 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
816 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
817 NFC_ECC_BLOCK_SIZE_MSK);
818 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
819 NFC_ECC_PIPELINE;
820
821 if (nand->ecc.size == 512)
822 ecc_ctl |= NFC_ECC_BLOCK_512;
823
824 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
825}
826
827static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
828{
829 struct nand_chip *nand = mtd_to_nand(mtd);
830 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
831
832 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
833 nfc->regs + NFC_REG_ECC_CTL);
834}
835
836static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
837{
838 buf[0] = user_data;
839 buf[1] = user_data >> 8;
840 buf[2] = user_data >> 16;
841 buf[3] = user_data >> 24;
842}
843
844static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
845{
846 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
847}
848
849static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
850 int step, bool bbm, int page)
851{
852 struct nand_chip *nand = mtd_to_nand(mtd);
853 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
854
855 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
856 oob);
857
858 /* De-randomize the Bad Block Marker. */
859 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
860 sunxi_nfc_randomize_bbm(mtd, page, oob);
861}
862
863static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
864 const u8 *oob, int step,
865 bool bbm, int page)
866{
867 struct nand_chip *nand = mtd_to_nand(mtd);
868 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
869 u8 user_data[4];
870
871 /* Randomize the Bad Block Marker. */
872 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
873 memcpy(user_data, oob, sizeof(user_data));
874 sunxi_nfc_randomize_bbm(mtd, page, user_data);
875 oob = user_data;
876 }
877
878 writel(sunxi_nfc_buf_to_user_data(oob),
879 nfc->regs + NFC_REG_USER_DATA(step));
880}
881
882static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
883 unsigned int *max_bitflips, int ret)
884{
885 if (ret < 0) {
886 mtd->ecc_stats.failed++;
887 } else {
888 mtd->ecc_stats.corrected += ret;
889 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
890 }
891}
892
893static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
894 int step, u32 status, bool *erased)
895{
896 struct nand_chip *nand = mtd_to_nand(mtd);
897 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
898 struct nand_ecc_ctrl *ecc = &nand->ecc;
899 u32 tmp;
900
901 *erased = false;
902
903 if (status & NFC_ECC_ERR(step))
904 return -EBADMSG;
905
906 if (status & NFC_ECC_PAT_FOUND(step)) {
907 u8 pattern;
908
909 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
910 pattern = 0x0;
911 } else {
912 pattern = 0xff;
913 *erased = true;
914 }
915
916 if (data)
917 memset(data, pattern, ecc->size);
918
919 if (oob)
920 memset(oob, pattern, ecc->bytes + 4);
921
922 return 0;
923 }
924
925 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
926
927 return NFC_ECC_ERR_CNT(step, tmp);
928}
929
930static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
931 u8 *data, int data_off,
932 u8 *oob, int oob_off,
933 int *cur_off,
934 unsigned int *max_bitflips,
935 bool bbm, bool oob_required, int page)
936{
937 struct nand_chip *nand = mtd_to_nand(mtd);
938 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
939 struct nand_ecc_ctrl *ecc = &nand->ecc;
940 int raw_mode = 0;
941 bool erased;
942 int ret;
943
944 if (*cur_off != data_off)
945 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
946
947 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
948
949 if (data_off + ecc->size != oob_off)
950 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
951
952 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
953 if (ret)
954 return ret;
955
956 sunxi_nfc_randomizer_enable(mtd);
957 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
958 nfc->regs + NFC_REG_CMD);
959
960 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
961 sunxi_nfc_randomizer_disable(mtd);
962 if (ret)
963 return ret;
964
965 *cur_off = oob_off + ecc->bytes + 4;
966
967 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
968 readl(nfc->regs + NFC_REG_ECC_ST),
969 &erased);
970 if (erased)
971 return 1;
972
973 if (ret < 0) {
974 /*
975 * Re-read the data with the randomizer disabled to identify
976 * bitflips in erased pages.
977 */
978 if (nand->options & NAND_NEED_SCRAMBLING) {
979 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
980 nand->read_buf(mtd, data, ecc->size);
981 } else {
982 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
983 ecc->size);
984 }
985
986 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
987 nand->read_buf(mtd, oob, ecc->bytes + 4);
988
989 ret = nand_check_erased_ecc_chunk(data, ecc->size,
990 oob, ecc->bytes + 4,
991 NULL, 0, ecc->strength);
992 if (ret >= 0)
993 raw_mode = 1;
994 } else {
995 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
996
997 if (oob_required) {
998 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
999 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1000 true, page);
1001
1002 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1003 bbm, page);
1004 }
1005 }
1006
1007 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1008
1009 return raw_mode;
1010}
1011
1012static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1013 u8 *oob, int *cur_off,
1014 bool randomize, int page)
1015{
1016 struct nand_chip *nand = mtd_to_nand(mtd);
1017 struct nand_ecc_ctrl *ecc = &nand->ecc;
1018 int offset = ((ecc->bytes + 4) * ecc->steps);
1019 int len = mtd->oobsize - offset;
1020
1021 if (len <= 0)
1022 return;
1023
1024 if (!cur_off || *cur_off != offset)
1025 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1026 offset + mtd->writesize, -1);
1027
1028 if (!randomize)
1029 sunxi_nfc_read_buf(mtd, oob + offset, len);
1030 else
1031 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1032 false, page);
1033
1034 if (cur_off)
1035 *cur_off = mtd->oobsize + mtd->writesize;
1036}
1037
1038static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1039 int oob_required, int page,
1040 int nchunks)
1041{
1042 struct nand_chip *nand = mtd_to_nand(mtd);
1043 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1044 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1045 struct nand_ecc_ctrl *ecc = &nand->ecc;
1046 unsigned int max_bitflips = 0;
1047 int ret, i, raw_mode = 0;
1048 struct scatterlist sg;
1049 u32 status;
1050
1051 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1052 if (ret)
1053 return ret;
1054
1055 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1056 DMA_FROM_DEVICE, &sg);
1057 if (ret)
1058 return ret;
1059
1060 sunxi_nfc_hw_ecc_enable(mtd);
1061 sunxi_nfc_randomizer_config(mtd, page, false);
1062 sunxi_nfc_randomizer_enable(mtd);
1063
1064 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1065 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1066
1067 dma_async_issue_pending(nfc->dmac);
1068
1069 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1070 nfc->regs + NFC_REG_CMD);
1071
1072 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1073 if (ret)
1074 dmaengine_terminate_all(nfc->dmac);
1075
1076 sunxi_nfc_randomizer_disable(mtd);
1077 sunxi_nfc_hw_ecc_disable(mtd);
1078
1079 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1080
1081 if (ret)
1082 return ret;
1083
1084 status = readl(nfc->regs + NFC_REG_ECC_ST);
1085
1086 for (i = 0; i < nchunks; i++) {
1087 int data_off = i * ecc->size;
1088 int oob_off = i * (ecc->bytes + 4);
1089 u8 *data = buf + data_off;
1090 u8 *oob = nand->oob_poi + oob_off;
1091 bool erased;
1092
1093 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1094 oob_required ? oob : NULL,
1095 i, status, &erased);
1096
1097 /* ECC errors are handled in the second loop. */
1098 if (ret < 0)
1099 continue;
1100
1101 if (oob_required && !erased) {
1102 /* TODO: use DMA to retrieve OOB */
1103 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1104 mtd->writesize + oob_off, -1);
1105 nand->read_buf(mtd, oob, ecc->bytes + 4);
1106
1107 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1108 !i, page);
1109 }
1110
1111 if (erased)
1112 raw_mode = 1;
1113
1114 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1115 }
1116
1117 if (status & NFC_ECC_ERR_MSK) {
1118 for (i = 0; i < nchunks; i++) {
1119 int data_off = i * ecc->size;
1120 int oob_off = i * (ecc->bytes + 4);
1121 u8 *data = buf + data_off;
1122 u8 *oob = nand->oob_poi + oob_off;
1123
1124 if (!(status & NFC_ECC_ERR(i)))
1125 continue;
1126
1127 /*
1128 * Re-read the data with the randomizer disabled to
1129 * identify bitflips in erased pages.
1130 */
1131 if (randomized) {
1132 /* TODO: use DMA to read page in raw mode */
1133 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1134 data_off, -1);
1135 nand->read_buf(mtd, data, ecc->size);
1136 }
1137
1138 /* TODO: use DMA to retrieve OOB */
1139 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1140 mtd->writesize + oob_off, -1);
1141 nand->read_buf(mtd, oob, ecc->bytes + 4);
1142
1143 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1144 oob, ecc->bytes + 4,
1145 NULL, 0,
1146 ecc->strength);
1147 if (ret >= 0)
1148 raw_mode = 1;
1149
1150 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1151 }
1152 }
1153
1154 if (oob_required)
1155 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1156 NULL, !raw_mode,
1157 page);
1158
1159 return max_bitflips;
1160}
1161
1162static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1163 const u8 *data, int data_off,
1164 const u8 *oob, int oob_off,
1165 int *cur_off, bool bbm,
1166 int page)
1167{
1168 struct nand_chip *nand = mtd_to_nand(mtd);
1169 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1170 struct nand_ecc_ctrl *ecc = &nand->ecc;
1171 int ret;
1172
1173 if (data_off != *cur_off)
1174 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1175
1176 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1177
1178 if (data_off + ecc->size != oob_off)
1179 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1180
1181 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1182 if (ret)
1183 return ret;
1184
1185 sunxi_nfc_randomizer_enable(mtd);
1186 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1187
1188 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1189 NFC_ACCESS_DIR | NFC_ECC_OP,
1190 nfc->regs + NFC_REG_CMD);
1191
1192 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1193 sunxi_nfc_randomizer_disable(mtd);
1194 if (ret)
1195 return ret;
1196
1197 *cur_off = oob_off + ecc->bytes + 4;
1198
1199 return 0;
1200}
1201
1202static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1203 u8 *oob, int *cur_off,
1204 int page)
1205{
1206 struct nand_chip *nand = mtd_to_nand(mtd);
1207 struct nand_ecc_ctrl *ecc = &nand->ecc;
1208 int offset = ((ecc->bytes + 4) * ecc->steps);
1209 int len = mtd->oobsize - offset;
1210
1211 if (len <= 0)
1212 return;
1213
1214 if (!cur_off || *cur_off != offset)
1215 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1216 offset + mtd->writesize, -1);
1217
1218 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1219
1220 if (cur_off)
1221 *cur_off = mtd->oobsize + mtd->writesize;
1222}
1223
1224static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1225 struct nand_chip *chip, uint8_t *buf,
1226 int oob_required, int page)
1227{
1228 struct nand_ecc_ctrl *ecc = &chip->ecc;
1229 unsigned int max_bitflips = 0;
1230 int ret, i, cur_off = 0;
1231 bool raw_mode = false;
1232
1233 sunxi_nfc_hw_ecc_enable(mtd);
1234
1235 for (i = 0; i < ecc->steps; i++) {
1236 int data_off = i * ecc->size;
1237 int oob_off = i * (ecc->bytes + 4);
1238 u8 *data = buf + data_off;
1239 u8 *oob = chip->oob_poi + oob_off;
1240
1241 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1242 oob_off + mtd->writesize,
1243 &cur_off, &max_bitflips,
1244 !i, oob_required, page);
1245 if (ret < 0)
1246 return ret;
1247 else if (ret)
1248 raw_mode = true;
1249 }
1250
1251 if (oob_required)
1252 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1253 !raw_mode, page);
1254
1255 sunxi_nfc_hw_ecc_disable(mtd);
1256
1257 return max_bitflips;
1258}
1259
1260static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1261 struct nand_chip *chip, u8 *buf,
1262 int oob_required, int page)
1263{
1264 int ret;
1265
1266 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1267 chip->ecc.steps);
1268 if (ret >= 0)
1269 return ret;
1270
1271 /* Fallback to PIO mode */
1272 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1273
1274 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1275}
1276
1277static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1278 struct nand_chip *chip,
1279 u32 data_offs, u32 readlen,
1280 u8 *bufpoi, int page)
1281{
1282 struct nand_ecc_ctrl *ecc = &chip->ecc;
1283 int ret, i, cur_off = 0;
1284 unsigned int max_bitflips = 0;
1285
1286 sunxi_nfc_hw_ecc_enable(mtd);
1287
1288 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1289 for (i = data_offs / ecc->size;
1290 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1291 int data_off = i * ecc->size;
1292 int oob_off = i * (ecc->bytes + 4);
1293 u8 *data = bufpoi + data_off;
1294 u8 *oob = chip->oob_poi + oob_off;
1295
1296 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1297 oob,
1298 oob_off + mtd->writesize,
1299 &cur_off, &max_bitflips, !i,
1300 false, page);
1301 if (ret < 0)
1302 return ret;
1303 }
1304
1305 sunxi_nfc_hw_ecc_disable(mtd);
1306
1307 return max_bitflips;
1308}
1309
1310static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1311 struct nand_chip *chip,
1312 u32 data_offs, u32 readlen,
1313 u8 *buf, int page)
1314{
1315 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1316 int ret;
1317
1318 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1319 if (ret >= 0)
1320 return ret;
1321
1322 /* Fallback to PIO mode */
1323 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1324
1325 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1326 buf, page);
1327}
1328
1329static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1330 struct nand_chip *chip,
1331 const uint8_t *buf, int oob_required,
1332 int page)
1333{
1334 struct nand_ecc_ctrl *ecc = &chip->ecc;
1335 int ret, i, cur_off = 0;
1336
1337 sunxi_nfc_hw_ecc_enable(mtd);
1338
1339 for (i = 0; i < ecc->steps; i++) {
1340 int data_off = i * ecc->size;
1341 int oob_off = i * (ecc->bytes + 4);
1342 const u8 *data = buf + data_off;
1343 const u8 *oob = chip->oob_poi + oob_off;
1344
1345 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1346 oob_off + mtd->writesize,
1347 &cur_off, !i, page);
1348 if (ret)
1349 return ret;
1350 }
1351
1352 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1353 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1354 &cur_off, page);
1355
1356 sunxi_nfc_hw_ecc_disable(mtd);
1357
1358 return 0;
1359}
1360
1361static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1362 struct nand_chip *chip,
1363 u32 data_offs, u32 data_len,
1364 const u8 *buf, int oob_required,
1365 int page)
1366{
1367 struct nand_ecc_ctrl *ecc = &chip->ecc;
1368 int ret, i, cur_off = 0;
1369
1370 sunxi_nfc_hw_ecc_enable(mtd);
1371
1372 for (i = data_offs / ecc->size;
1373 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1374 int data_off = i * ecc->size;
1375 int oob_off = i * (ecc->bytes + 4);
1376 const u8 *data = buf + data_off;
1377 const u8 *oob = chip->oob_poi + oob_off;
1378
1379 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1380 oob_off + mtd->writesize,
1381 &cur_off, !i, page);
1382 if (ret)
1383 return ret;
1384 }
1385
1386 sunxi_nfc_hw_ecc_disable(mtd);
1387
1388 return 0;
1389}
1390
1391static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1392 struct nand_chip *chip,
1393 const u8 *buf,
1394 int oob_required,
1395 int page)
1396{
1397 struct nand_chip *nand = mtd_to_nand(mtd);
1398 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1399 struct nand_ecc_ctrl *ecc = &nand->ecc;
1400 struct scatterlist sg;
1401 int ret, i;
1402
1403 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1404 if (ret)
1405 return ret;
1406
1407 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1408 DMA_TO_DEVICE, &sg);
1409 if (ret)
1410 goto pio_fallback;
1411
1412 for (i = 0; i < ecc->steps; i++) {
1413 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1414
1415 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1416 }
1417
1418 sunxi_nfc_hw_ecc_enable(mtd);
1419 sunxi_nfc_randomizer_config(mtd, page, false);
1420 sunxi_nfc_randomizer_enable(mtd);
1421
1422 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1423 nfc->regs + NFC_REG_RCMD_SET);
1424
1425 dma_async_issue_pending(nfc->dmac);
1426
1427 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1428 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1429 nfc->regs + NFC_REG_CMD);
1430
1431 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1432 if (ret)
1433 dmaengine_terminate_all(nfc->dmac);
1434
1435 sunxi_nfc_randomizer_disable(mtd);
1436 sunxi_nfc_hw_ecc_disable(mtd);
1437
1438 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1439
1440 if (ret)
1441 return ret;
1442
1443 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1444 /* TODO: use DMA to transfer extra OOB bytes ? */
1445 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1446 NULL, page);
1447
1448 return 0;
1449
1450pio_fallback:
1451 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1452}
1453
1454static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1455 struct nand_chip *chip,
1456 uint8_t *buf, int oob_required,
1457 int page)
1458{
1459 struct nand_ecc_ctrl *ecc = &chip->ecc;
1460 unsigned int max_bitflips = 0;
1461 int ret, i, cur_off = 0;
1462 bool raw_mode = false;
1463
1464 sunxi_nfc_hw_ecc_enable(mtd);
1465
1466 for (i = 0; i < ecc->steps; i++) {
1467 int data_off = i * (ecc->size + ecc->bytes + 4);
1468 int oob_off = data_off + ecc->size;
1469 u8 *data = buf + (i * ecc->size);
1470 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1471
1472 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1473 oob_off, &cur_off,
1474 &max_bitflips, !i,
1475 oob_required,
1476 page);
1477 if (ret < 0)
1478 return ret;
1479 else if (ret)
1480 raw_mode = true;
1481 }
1482
1483 if (oob_required)
1484 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1485 !raw_mode, page);
1486
1487 sunxi_nfc_hw_ecc_disable(mtd);
1488
1489 return max_bitflips;
1490}
1491
1492static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1493 struct nand_chip *chip,
1494 const uint8_t *buf,
1495 int oob_required, int page)
1496{
1497 struct nand_ecc_ctrl *ecc = &chip->ecc;
1498 int ret, i, cur_off = 0;
1499
1500 sunxi_nfc_hw_ecc_enable(mtd);
1501
1502 for (i = 0; i < ecc->steps; i++) {
1503 int data_off = i * (ecc->size + ecc->bytes + 4);
1504 int oob_off = data_off + ecc->size;
1505 const u8 *data = buf + (i * ecc->size);
1506 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1507
1508 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1509 oob, oob_off, &cur_off,
1510 false, page);
1511 if (ret)
1512 return ret;
1513 }
1514
1515 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1516 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1517 &cur_off, page);
1518
1519 sunxi_nfc_hw_ecc_disable(mtd);
1520
1521 return 0;
1522}
1523
1524static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1525 struct nand_chip *chip,
1526 int page)
1527{
1528 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1529
1530 chip->pagebuf = -1;
1531
1532 return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1533}
1534
1535static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1536 struct nand_chip *chip,
1537 int page)
1538{
1539 int ret, status;
1540
1541 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1542
1543 chip->pagebuf = -1;
1544
1545 memset(chip->buffers->databuf, 0xff, mtd->writesize);
1546 ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1547 if (ret)
1548 return ret;
1549
1550 /* Send command to program the OOB data */
1551 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1552
1553 status = chip->waitfunc(mtd, chip);
1554
1555 return status & NAND_STATUS_FAIL ? -EIO : 0;
1556}
1557
1558static const s32 tWB_lut[] = {6, 12, 16, 20};
1559static const s32 tRHW_lut[] = {4, 8, 12, 20};
1560
1561static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1562 u32 clk_period)
1563{
1564 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1565 int i;
1566
1567 for (i = 0; i < lut_size; i++) {
1568 if (clk_cycles <= lut[i])
1569 return i;
1570 }
1571
1572 /* Doesn't fit */
1573 return -EINVAL;
1574}
1575
1576#define sunxi_nand_lookup_timing(l, p, c) \
1577 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1578
1579static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1580 const struct nand_data_interface *conf,
1581 bool check_only)
1582{
1583 struct nand_chip *nand = mtd_to_nand(mtd);
1584 struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1585 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1586 const struct nand_sdr_timings *timings;
1587 u32 min_clk_period = 0;
1588 s32 tWB, tADL, tWHR, tRHW, tCAD;
1589 long real_clk_rate;
1590
1591 timings = nand_get_sdr_timings(conf);
1592 if (IS_ERR(timings))
1593 return -ENOTSUPP;
1594
1595 /* T1 <=> tCLS */
1596 if (timings->tCLS_min > min_clk_period)
1597 min_clk_period = timings->tCLS_min;
1598
1599 /* T2 <=> tCLH */
1600 if (timings->tCLH_min > min_clk_period)
1601 min_clk_period = timings->tCLH_min;
1602
1603 /* T3 <=> tCS */
1604 if (timings->tCS_min > min_clk_period)
1605 min_clk_period = timings->tCS_min;
1606
1607 /* T4 <=> tCH */
1608 if (timings->tCH_min > min_clk_period)
1609 min_clk_period = timings->tCH_min;
1610
1611 /* T5 <=> tWP */
1612 if (timings->tWP_min > min_clk_period)
1613 min_clk_period = timings->tWP_min;
1614
1615 /* T6 <=> tWH */
1616 if (timings->tWH_min > min_clk_period)
1617 min_clk_period = timings->tWH_min;
1618
1619 /* T7 <=> tALS */
1620 if (timings->tALS_min > min_clk_period)
1621 min_clk_period = timings->tALS_min;
1622
1623 /* T8 <=> tDS */
1624 if (timings->tDS_min > min_clk_period)
1625 min_clk_period = timings->tDS_min;
1626
1627 /* T9 <=> tDH */
1628 if (timings->tDH_min > min_clk_period)
1629 min_clk_period = timings->tDH_min;
1630
1631 /* T10 <=> tRR */
1632 if (timings->tRR_min > (min_clk_period * 3))
1633 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1634
1635 /* T11 <=> tALH */
1636 if (timings->tALH_min > min_clk_period)
1637 min_clk_period = timings->tALH_min;
1638
1639 /* T12 <=> tRP */
1640 if (timings->tRP_min > min_clk_period)
1641 min_clk_period = timings->tRP_min;
1642
1643 /* T13 <=> tREH */
1644 if (timings->tREH_min > min_clk_period)
1645 min_clk_period = timings->tREH_min;
1646
1647 /* T14 <=> tRC */
1648 if (timings->tRC_min > (min_clk_period * 2))
1649 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1650
1651 /* T15 <=> tWC */
1652 if (timings->tWC_min > (min_clk_period * 2))
1653 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1654
1655 /* T16 - T19 + tCAD */
1656 if (timings->tWB_max > (min_clk_period * 20))
1657 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1658
1659 if (timings->tADL_min > (min_clk_period * 32))
1660 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1661
1662 if (timings->tWHR_min > (min_clk_period * 32))
1663 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1664
1665 if (timings->tRHW_min > (min_clk_period * 20))
1666 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1667
1668 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1669 min_clk_period);
1670 if (tWB < 0) {
1671 dev_err(nfc->dev, "unsupported tWB\n");
1672 return tWB;
1673 }
1674
1675 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1676 if (tADL > 3) {
1677 dev_err(nfc->dev, "unsupported tADL\n");
1678 return -EINVAL;
1679 }
1680
1681 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1682 if (tWHR > 3) {
1683 dev_err(nfc->dev, "unsupported tWHR\n");
1684 return -EINVAL;
1685 }
1686
1687 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1688 min_clk_period);
1689 if (tRHW < 0) {
1690 dev_err(nfc->dev, "unsupported tRHW\n");
1691 return tRHW;
1692 }
1693
1694 if (check_only)
1695 return 0;
1696
1697 /*
1698 * TODO: according to ONFI specs this value only applies for DDR NAND,
1699 * but Allwinner seems to set this to 0x7. Mimic them for now.
1700 */
1701 tCAD = 0x7;
1702
1703 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1704 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1705
1706 /* Convert min_clk_period from picoseconds to nanoseconds */
1707 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1708
1709 /*
1710 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1711 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1712 * This new formula was verified with a scope and validated by
1713 * Allwinner engineers.
1714 */
1715 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1716 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1717
1718 /*
1719 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1720 * output cycle timings shall be used if the host drives tRC less than
1721 * 30 ns.
1722 */
1723 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1724 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1725 NFC_TIMING_CTL_EDO : 0;
1726
1727 return 0;
1728}
1729
1730static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1731 struct mtd_oob_region *oobregion)
1732{
1733 struct nand_chip *nand = mtd_to_nand(mtd);
1734 struct nand_ecc_ctrl *ecc = &nand->ecc;
1735
1736 if (section >= ecc->steps)
1737 return -ERANGE;
1738
1739 oobregion->offset = section * (ecc->bytes + 4) + 4;
1740 oobregion->length = ecc->bytes;
1741
1742 return 0;
1743}
1744
1745static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1746 struct mtd_oob_region *oobregion)
1747{
1748 struct nand_chip *nand = mtd_to_nand(mtd);
1749 struct nand_ecc_ctrl *ecc = &nand->ecc;
1750
1751 if (section > ecc->steps)
1752 return -ERANGE;
1753
1754 /*
1755 * The first 2 bytes are used for BB markers, hence we
1756 * only have 2 bytes available in the first user data
1757 * section.
1758 */
1759 if (!section && ecc->mode == NAND_ECC_HW) {
1760 oobregion->offset = 2;
1761 oobregion->length = 2;
1762
1763 return 0;
1764 }
1765
1766 oobregion->offset = section * (ecc->bytes + 4);
1767
1768 if (section < ecc->steps)
1769 oobregion->length = 4;
1770 else
1771 oobregion->offset = mtd->oobsize - oobregion->offset;
1772
1773 return 0;
1774}
1775
1776static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1777 .ecc = sunxi_nand_ooblayout_ecc,
1778 .free = sunxi_nand_ooblayout_free,
1779};
1780
1781static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1782 struct nand_ecc_ctrl *ecc,
1783 struct device_node *np)
1784{
1785 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1786 struct nand_chip *nand = mtd_to_nand(mtd);
1787 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1788 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1789 struct sunxi_nand_hw_ecc *data;
1790 int nsectors;
1791 int ret;
1792 int i;
1793
1794 if (ecc->options & NAND_ECC_MAXIMIZE) {
1795 int bytes;
1796
1797 ecc->size = 1024;
1798 nsectors = mtd->writesize / ecc->size;
1799
1800 /* Reserve 2 bytes for the BBM */
1801 bytes = (mtd->oobsize - 2) / nsectors;
1802
1803 /* 4 non-ECC bytes are added before each ECC bytes section */
1804 bytes -= 4;
1805
1806 /* and bytes has to be even. */
1807 if (bytes % 2)
1808 bytes--;
1809
1810 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1811
1812 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1813 if (strengths[i] > ecc->strength)
1814 break;
1815 }
1816
1817 if (!i)
1818 ecc->strength = 0;
1819 else
1820 ecc->strength = strengths[i - 1];
1821 }
1822
1823 if (ecc->size != 512 && ecc->size != 1024)
1824 return -EINVAL;
1825
1826 data = kzalloc(sizeof(*data), GFP_KERNEL);
1827 if (!data)
1828 return -ENOMEM;
1829
1830 /* Prefer 1k ECC chunk over 512 ones */
1831 if (ecc->size == 512 && mtd->writesize > 512) {
1832 ecc->size = 1024;
1833 ecc->strength *= 2;
1834 }
1835
1836 /* Add ECC info retrieval from DT */
1837 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1838 if (ecc->strength <= strengths[i])
1839 break;
1840 }
1841
1842 if (i >= ARRAY_SIZE(strengths)) {
1843 dev_err(nfc->dev, "unsupported strength\n");
1844 ret = -ENOTSUPP;
1845 goto err;
1846 }
1847
1848 data->mode = i;
1849
1850 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1851 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1852
1853 /* HW ECC always work with even numbers of ECC bytes */
1854 ecc->bytes = ALIGN(ecc->bytes, 2);
1855
1856 nsectors = mtd->writesize / ecc->size;
1857
1858 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1859 ret = -EINVAL;
1860 goto err;
1861 }
1862
1863 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1864 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1865 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1866 ecc->priv = data;
1867
1868 return 0;
1869
1870err:
1871 kfree(data);
1872
1873 return ret;
1874}
1875
1876static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1877{
1878 kfree(ecc->priv);
1879}
1880
1881static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1882 struct nand_ecc_ctrl *ecc,
1883 struct device_node *np)
1884{
1885 struct nand_chip *nand = mtd_to_nand(mtd);
1886 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1887 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1888 int ret;
1889
1890 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1891 if (ret)
1892 return ret;
1893
1894 if (nfc->dmac) {
1895 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1896 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1897 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1898 nand->options |= NAND_USE_BOUNCE_BUFFER;
1899 } else {
1900 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1901 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1902 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1903 }
1904
1905 /* TODO: support DMA for raw accesses and subpage write */
1906 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1907 ecc->read_oob_raw = nand_read_oob_std;
1908 ecc->write_oob_raw = nand_write_oob_std;
1909 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1910
1911 return 0;
1912}
1913
1914static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1915 struct nand_ecc_ctrl *ecc,
1916 struct device_node *np)
1917{
1918 int ret;
1919
1920 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1921 if (ret)
1922 return ret;
1923
1924 ecc->prepad = 4;
1925 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1926 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1927 ecc->read_oob_raw = nand_read_oob_syndrome;
1928 ecc->write_oob_raw = nand_write_oob_syndrome;
1929
1930 return 0;
1931}
1932
1933static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1934{
1935 switch (ecc->mode) {
1936 case NAND_ECC_HW:
1937 case NAND_ECC_HW_SYNDROME:
1938 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1939 break;
1940 case NAND_ECC_NONE:
1941 default:
1942 break;
1943 }
1944}
1945
1946static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1947 struct device_node *np)
1948{
1949 struct nand_chip *nand = mtd_to_nand(mtd);
1950 int ret;
1951
1952 if (!ecc->size) {
1953 ecc->size = nand->ecc_step_ds;
1954 ecc->strength = nand->ecc_strength_ds;
1955 }
1956
1957 if (!ecc->size || !ecc->strength)
1958 return -EINVAL;
1959
1960 switch (ecc->mode) {
1961 case NAND_ECC_HW:
1962 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1963 if (ret)
1964 return ret;
1965 break;
1966 case NAND_ECC_HW_SYNDROME:
1967 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1968 if (ret)
1969 return ret;
1970 break;
1971 case NAND_ECC_NONE:
1972 case NAND_ECC_SOFT:
1973 break;
1974 default:
1975 return -EINVAL;
1976 }
1977
1978 return 0;
1979}
1980
1981static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1982 struct device_node *np)
1983{
1984 struct sunxi_nand_chip *chip;
1985 struct mtd_info *mtd;
1986 struct nand_chip *nand;
1987 int nsels;
1988 int ret;
1989 int i;
1990 u32 tmp;
1991
1992 if (!of_get_property(np, "reg", &nsels))
1993 return -EINVAL;
1994
1995 nsels /= sizeof(u32);
1996 if (!nsels) {
1997 dev_err(dev, "invalid reg property size\n");
1998 return -EINVAL;
1999 }
2000
2001 chip = devm_kzalloc(dev,
2002 sizeof(*chip) +
2003 (nsels * sizeof(struct sunxi_nand_chip_sel)),
2004 GFP_KERNEL);
2005 if (!chip) {
2006 dev_err(dev, "could not allocate chip\n");
2007 return -ENOMEM;
2008 }
2009
2010 chip->nsels = nsels;
2011 chip->selected = -1;
2012
2013 for (i = 0; i < nsels; i++) {
2014 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2015 if (ret) {
2016 dev_err(dev, "could not retrieve reg property: %d\n",
2017 ret);
2018 return ret;
2019 }
2020
2021 if (tmp > NFC_MAX_CS) {
2022 dev_err(dev,
2023 "invalid reg value: %u (max CS = 7)\n",
2024 tmp);
2025 return -EINVAL;
2026 }
2027
2028 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2029 dev_err(dev, "CS %d already assigned\n", tmp);
2030 return -EINVAL;
2031 }
2032
2033 chip->sels[i].cs = tmp;
2034
2035 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2036 tmp < 2) {
2037 chip->sels[i].rb.type = RB_NATIVE;
2038 chip->sels[i].rb.info.nativeid = tmp;
2039 } else {
2040 ret = of_get_named_gpio(np, "rb-gpios", i);
2041 if (ret >= 0) {
2042 tmp = ret;
2043 chip->sels[i].rb.type = RB_GPIO;
2044 chip->sels[i].rb.info.gpio = tmp;
2045 ret = devm_gpio_request(dev, tmp, "nand-rb");
2046 if (ret)
2047 return ret;
2048
2049 ret = gpio_direction_input(tmp);
2050 if (ret)
2051 return ret;
2052 } else {
2053 chip->sels[i].rb.type = RB_NONE;
2054 }
2055 }
2056 }
2057
2058 nand = &chip->nand;
2059 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2060 nand->chip_delay = 200;
2061 nand->controller = &nfc->controller;
2062 /*
2063 * Set the ECC mode to the default value in case nothing is specified
2064 * in the DT.
2065 */
2066 nand->ecc.mode = NAND_ECC_HW;
2067 nand_set_flash_node(nand, np);
2068 nand->select_chip = sunxi_nfc_select_chip;
2069 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2070 nand->read_buf = sunxi_nfc_read_buf;
2071 nand->write_buf = sunxi_nfc_write_buf;
2072 nand->read_byte = sunxi_nfc_read_byte;
2073 nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2074
2075 mtd = nand_to_mtd(nand);
2076 mtd->dev.parent = dev;
2077
2078 ret = nand_scan_ident(mtd, nsels, NULL);
2079 if (ret)
2080 return ret;
2081
2082 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2083 nand->bbt_options |= NAND_BBT_NO_OOB;
2084
2085 if (nand->options & NAND_NEED_SCRAMBLING)
2086 nand->options |= NAND_NO_SUBPAGE_WRITE;
2087
2088 nand->options |= NAND_SUBPAGE_READ;
2089
2090 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2091 if (ret) {
2092 dev_err(dev, "ECC init failed: %d\n", ret);
2093 return ret;
2094 }
2095
2096 ret = nand_scan_tail(mtd);
2097 if (ret) {
2098 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2099 return ret;
2100 }
2101
2102 ret = mtd_device_register(mtd, NULL, 0);
2103 if (ret) {
2104 dev_err(dev, "failed to register mtd device: %d\n", ret);
2105 nand_release(mtd);
2106 return ret;
2107 }
2108
2109 list_add_tail(&chip->node, &nfc->chips);
2110
2111 return 0;
2112}
2113
2114static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2115{
2116 struct device_node *np = dev->of_node;
2117 struct device_node *nand_np;
2118 int nchips = of_get_child_count(np);
2119 int ret;
2120
2121 if (nchips > 8) {
2122 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2123 return -EINVAL;
2124 }
2125
2126 for_each_child_of_node(np, nand_np) {
2127 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2128 if (ret) {
2129 of_node_put(nand_np);
2130 return ret;
2131 }
2132 }
2133
2134 return 0;
2135}
2136
2137static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2138{
2139 struct sunxi_nand_chip *chip;
2140
2141 while (!list_empty(&nfc->chips)) {
2142 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2143 node);
2144 nand_release(nand_to_mtd(&chip->nand));
2145 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2146 list_del(&chip->node);
2147 }
2148}
2149
2150static int sunxi_nfc_probe(struct platform_device *pdev)
2151{
2152 struct device *dev = &pdev->dev;
2153 struct resource *r;
2154 struct sunxi_nfc *nfc;
2155 int irq;
2156 int ret;
2157
2158 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2159 if (!nfc)
2160 return -ENOMEM;
2161
2162 nfc->dev = dev;
2163 nand_hw_control_init(&nfc->controller);
2164 INIT_LIST_HEAD(&nfc->chips);
2165
2166 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2167 nfc->regs = devm_ioremap_resource(dev, r);
2168 if (IS_ERR(nfc->regs))
2169 return PTR_ERR(nfc->regs);
2170
2171 irq = platform_get_irq(pdev, 0);
2172 if (irq < 0) {
2173 dev_err(dev, "failed to retrieve irq\n");
2174 return irq;
2175 }
2176
2177 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2178 if (IS_ERR(nfc->ahb_clk)) {
2179 dev_err(dev, "failed to retrieve ahb clk\n");
2180 return PTR_ERR(nfc->ahb_clk);
2181 }
2182
2183 ret = clk_prepare_enable(nfc->ahb_clk);
2184 if (ret)
2185 return ret;
2186
2187 nfc->mod_clk = devm_clk_get(dev, "mod");
2188 if (IS_ERR(nfc->mod_clk)) {
2189 dev_err(dev, "failed to retrieve mod clk\n");
2190 ret = PTR_ERR(nfc->mod_clk);
2191 goto out_ahb_clk_unprepare;
2192 }
2193
2194 ret = clk_prepare_enable(nfc->mod_clk);
2195 if (ret)
2196 goto out_ahb_clk_unprepare;
2197
2198 nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2199 if (!IS_ERR(nfc->reset)) {
2200 ret = reset_control_deassert(nfc->reset);
2201 if (ret) {
2202 dev_err(dev, "reset err %d\n", ret);
2203 goto out_mod_clk_unprepare;
2204 }
2205 } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2206 ret = PTR_ERR(nfc->reset);
2207 goto out_mod_clk_unprepare;
2208 }
2209
2210 ret = sunxi_nfc_rst(nfc);
2211 if (ret)
2212 goto out_ahb_reset_reassert;
2213
2214 writel(0, nfc->regs + NFC_REG_INT);
2215 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2216 0, "sunxi-nand", nfc);
2217 if (ret)
2218 goto out_ahb_reset_reassert;
2219
2220 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2221 if (nfc->dmac) {
2222 struct dma_slave_config dmac_cfg = { };
2223
2224 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2225 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2226 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2227 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2228 dmac_cfg.src_maxburst = 4;
2229 dmac_cfg.dst_maxburst = 4;
2230 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2231 } else {
2232 dev_warn(dev, "failed to request rxtx DMA channel\n");
2233 }
2234
2235 platform_set_drvdata(pdev, nfc);
2236
2237 ret = sunxi_nand_chips_init(dev, nfc);
2238 if (ret) {
2239 dev_err(dev, "failed to init nand chips\n");
2240 goto out_release_dmac;
2241 }
2242
2243 return 0;
2244
2245out_release_dmac:
2246 if (nfc->dmac)
2247 dma_release_channel(nfc->dmac);
2248out_ahb_reset_reassert:
2249 if (!IS_ERR(nfc->reset))
2250 reset_control_assert(nfc->reset);
2251out_mod_clk_unprepare:
2252 clk_disable_unprepare(nfc->mod_clk);
2253out_ahb_clk_unprepare:
2254 clk_disable_unprepare(nfc->ahb_clk);
2255
2256 return ret;
2257}
2258
2259static int sunxi_nfc_remove(struct platform_device *pdev)
2260{
2261 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2262
2263 sunxi_nand_chips_cleanup(nfc);
2264
2265 if (!IS_ERR(nfc->reset))
2266 reset_control_assert(nfc->reset);
2267
2268 if (nfc->dmac)
2269 dma_release_channel(nfc->dmac);
2270 clk_disable_unprepare(nfc->mod_clk);
2271 clk_disable_unprepare(nfc->ahb_clk);
2272
2273 return 0;
2274}
2275
2276static const struct of_device_id sunxi_nfc_ids[] = {
2277 { .compatible = "allwinner,sun4i-a10-nand" },
2278 { /* sentinel */ }
2279};
2280MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2281
2282static struct platform_driver sunxi_nfc_driver = {
2283 .driver = {
2284 .name = "sunxi_nand",
2285 .of_match_table = sunxi_nfc_ids,
2286 },
2287 .probe = sunxi_nfc_probe,
2288 .remove = sunxi_nfc_remove,
2289};
2290module_platform_driver(sunxi_nfc_driver);
2291
2292MODULE_LICENSE("GPL v2");
2293MODULE_AUTHOR("Boris BREZILLON");
2294MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2295MODULE_ALIAS("platform:sunxi_nand");