Loading...
Note: File does not exist in v5.9.
1/*
2 * Copyright © 2003 Rick Bronson
3 *
4 * Derived from drivers/mtd/nand/autcpu12.c
5 * Copyright © 2001 Thomas Gleixner (gleixner@autronix.de)
6 *
7 * Derived from drivers/mtd/spia.c
8 * Copyright © 2000 Steven J. Hill (sjhill@cotw.com)
9 *
10 *
11 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
12 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright © 2007
13 *
14 * Derived from Das U-Boot source code
15 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
16 * © Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
17 *
18 * Add Programmable Multibit ECC support for various AT91 SoC
19 * © Copyright 2012 ATMEL, Hong Xu
20 *
21 * Add Nand Flash Controller support for SAMA5 SoC
22 * © Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
23 *
24 * This program is free software; you can redistribute it and/or modify
25 * it under the terms of the GNU General Public License version 2 as
26 * published by the Free Software Foundation.
27 *
28 */
29
30#include <linux/clk.h>
31#include <linux/dma-mapping.h>
32#include <linux/slab.h>
33#include <linux/module.h>
34#include <linux/moduleparam.h>
35#include <linux/platform_device.h>
36#include <linux/of.h>
37#include <linux/of_device.h>
38#include <linux/of_gpio.h>
39#include <linux/of_mtd.h>
40#include <linux/mtd/mtd.h>
41#include <linux/mtd/nand.h>
42#include <linux/mtd/partitions.h>
43
44#include <linux/delay.h>
45#include <linux/dmaengine.h>
46#include <linux/gpio.h>
47#include <linux/interrupt.h>
48#include <linux/io.h>
49#include <linux/platform_data/atmel.h>
50
51static int use_dma = 1;
52module_param(use_dma, int, 0);
53
54static int on_flash_bbt = 0;
55module_param(on_flash_bbt, int, 0);
56
57/* Register access macros */
58#define ecc_readl(add, reg) \
59 __raw_readl(add + ATMEL_ECC_##reg)
60#define ecc_writel(add, reg, value) \
61 __raw_writel((value), add + ATMEL_ECC_##reg)
62
63#include "atmel_nand_ecc.h" /* Hardware ECC registers */
64#include "atmel_nand_nfc.h" /* Nand Flash Controller definition */
65
66struct atmel_nand_caps {
67 bool pmecc_correct_erase_page;
68 uint8_t pmecc_max_correction;
69};
70
71struct atmel_nand_nfc_caps {
72 uint32_t rb_mask;
73};
74
75/* oob layout for large page size
76 * bad block info is on bytes 0 and 1
77 * the bytes have to be consecutives to avoid
78 * several NAND_CMD_RNDOUT during read
79 */
80static struct nand_ecclayout atmel_oobinfo_large = {
81 .eccbytes = 4,
82 .eccpos = {60, 61, 62, 63},
83 .oobfree = {
84 {2, 58}
85 },
86};
87
88/* oob layout for small page size
89 * bad block info is on bytes 4 and 5
90 * the bytes have to be consecutives to avoid
91 * several NAND_CMD_RNDOUT during read
92 */
93static struct nand_ecclayout atmel_oobinfo_small = {
94 .eccbytes = 4,
95 .eccpos = {0, 1, 2, 3},
96 .oobfree = {
97 {6, 10}
98 },
99};
100
101struct atmel_nfc {
102 void __iomem *base_cmd_regs;
103 void __iomem *hsmc_regs;
104 void *sram_bank0;
105 dma_addr_t sram_bank0_phys;
106 bool use_nfc_sram;
107 bool write_by_sram;
108
109 struct clk *clk;
110
111 bool is_initialized;
112 struct completion comp_ready;
113 struct completion comp_cmd_done;
114 struct completion comp_xfer_done;
115
116 /* Point to the sram bank which include readed data via NFC */
117 void *data_in_sram;
118 bool will_write_sram;
119 const struct atmel_nand_nfc_caps *caps;
120};
121static struct atmel_nfc nand_nfc;
122
123struct atmel_nand_host {
124 struct nand_chip nand_chip;
125 void __iomem *io_base;
126 dma_addr_t io_phys;
127 struct atmel_nand_data board;
128 struct device *dev;
129 void __iomem *ecc;
130
131 struct completion comp;
132 struct dma_chan *dma_chan;
133
134 struct atmel_nfc *nfc;
135
136 const struct atmel_nand_caps *caps;
137 bool has_pmecc;
138 u8 pmecc_corr_cap;
139 u16 pmecc_sector_size;
140 bool has_no_lookup_table;
141 u32 pmecc_lookup_table_offset;
142 u32 pmecc_lookup_table_offset_512;
143 u32 pmecc_lookup_table_offset_1024;
144
145 int pmecc_degree; /* Degree of remainders */
146 int pmecc_cw_len; /* Length of codeword */
147
148 void __iomem *pmerrloc_base;
149 void __iomem *pmerrloc_el_base;
150 void __iomem *pmecc_rom_base;
151
152 /* lookup table for alpha_to and index_of */
153 void __iomem *pmecc_alpha_to;
154 void __iomem *pmecc_index_of;
155
156 /* data for pmecc computation */
157 int16_t *pmecc_partial_syn;
158 int16_t *pmecc_si;
159 int16_t *pmecc_smu; /* Sigma table */
160 int16_t *pmecc_lmu; /* polynomal order */
161 int *pmecc_mu;
162 int *pmecc_dmu;
163 int *pmecc_delta;
164};
165
166static struct nand_ecclayout atmel_pmecc_oobinfo;
167
168/*
169 * Enable NAND.
170 */
171static void atmel_nand_enable(struct atmel_nand_host *host)
172{
173 if (gpio_is_valid(host->board.enable_pin))
174 gpio_set_value(host->board.enable_pin, 0);
175}
176
177/*
178 * Disable NAND.
179 */
180static void atmel_nand_disable(struct atmel_nand_host *host)
181{
182 if (gpio_is_valid(host->board.enable_pin))
183 gpio_set_value(host->board.enable_pin, 1);
184}
185
186/*
187 * Hardware specific access to control-lines
188 */
189static void atmel_nand_cmd_ctrl(struct mtd_info *mtd, int cmd, unsigned int ctrl)
190{
191 struct nand_chip *nand_chip = mtd_to_nand(mtd);
192 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
193
194 if (ctrl & NAND_CTRL_CHANGE) {
195 if (ctrl & NAND_NCE)
196 atmel_nand_enable(host);
197 else
198 atmel_nand_disable(host);
199 }
200 if (cmd == NAND_CMD_NONE)
201 return;
202
203 if (ctrl & NAND_CLE)
204 writeb(cmd, host->io_base + (1 << host->board.cle));
205 else
206 writeb(cmd, host->io_base + (1 << host->board.ale));
207}
208
209/*
210 * Read the Device Ready pin.
211 */
212static int atmel_nand_device_ready(struct mtd_info *mtd)
213{
214 struct nand_chip *nand_chip = mtd_to_nand(mtd);
215 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
216
217 return gpio_get_value(host->board.rdy_pin) ^
218 !!host->board.rdy_pin_active_low;
219}
220
221/* Set up for hardware ready pin and enable pin. */
222static int atmel_nand_set_enable_ready_pins(struct mtd_info *mtd)
223{
224 struct nand_chip *chip = mtd_to_nand(mtd);
225 struct atmel_nand_host *host = nand_get_controller_data(chip);
226 int res = 0;
227
228 if (gpio_is_valid(host->board.rdy_pin)) {
229 res = devm_gpio_request(host->dev,
230 host->board.rdy_pin, "nand_rdy");
231 if (res < 0) {
232 dev_err(host->dev,
233 "can't request rdy gpio %d\n",
234 host->board.rdy_pin);
235 return res;
236 }
237
238 res = gpio_direction_input(host->board.rdy_pin);
239 if (res < 0) {
240 dev_err(host->dev,
241 "can't request input direction rdy gpio %d\n",
242 host->board.rdy_pin);
243 return res;
244 }
245
246 chip->dev_ready = atmel_nand_device_ready;
247 }
248
249 if (gpio_is_valid(host->board.enable_pin)) {
250 res = devm_gpio_request(host->dev,
251 host->board.enable_pin, "nand_enable");
252 if (res < 0) {
253 dev_err(host->dev,
254 "can't request enable gpio %d\n",
255 host->board.enable_pin);
256 return res;
257 }
258
259 res = gpio_direction_output(host->board.enable_pin, 1);
260 if (res < 0) {
261 dev_err(host->dev,
262 "can't request output direction enable gpio %d\n",
263 host->board.enable_pin);
264 return res;
265 }
266 }
267
268 return res;
269}
270
271/*
272 * Minimal-overhead PIO for data access.
273 */
274static void atmel_read_buf8(struct mtd_info *mtd, u8 *buf, int len)
275{
276 struct nand_chip *nand_chip = mtd_to_nand(mtd);
277 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
278
279 if (host->nfc && host->nfc->use_nfc_sram && host->nfc->data_in_sram) {
280 memcpy(buf, host->nfc->data_in_sram, len);
281 host->nfc->data_in_sram += len;
282 } else {
283 __raw_readsb(nand_chip->IO_ADDR_R, buf, len);
284 }
285}
286
287static void atmel_read_buf16(struct mtd_info *mtd, u8 *buf, int len)
288{
289 struct nand_chip *nand_chip = mtd_to_nand(mtd);
290 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
291
292 if (host->nfc && host->nfc->use_nfc_sram && host->nfc->data_in_sram) {
293 memcpy(buf, host->nfc->data_in_sram, len);
294 host->nfc->data_in_sram += len;
295 } else {
296 __raw_readsw(nand_chip->IO_ADDR_R, buf, len / 2);
297 }
298}
299
300static void atmel_write_buf8(struct mtd_info *mtd, const u8 *buf, int len)
301{
302 struct nand_chip *nand_chip = mtd_to_nand(mtd);
303
304 __raw_writesb(nand_chip->IO_ADDR_W, buf, len);
305}
306
307static void atmel_write_buf16(struct mtd_info *mtd, const u8 *buf, int len)
308{
309 struct nand_chip *nand_chip = mtd_to_nand(mtd);
310
311 __raw_writesw(nand_chip->IO_ADDR_W, buf, len / 2);
312}
313
314static void dma_complete_func(void *completion)
315{
316 complete(completion);
317}
318
319static int nfc_set_sram_bank(struct atmel_nand_host *host, unsigned int bank)
320{
321 /* NFC only has two banks. Must be 0 or 1 */
322 if (bank > 1)
323 return -EINVAL;
324
325 if (bank) {
326 struct mtd_info *mtd = nand_to_mtd(&host->nand_chip);
327
328 /* Only for a 2k-page or lower flash, NFC can handle 2 banks */
329 if (mtd->writesize > 2048)
330 return -EINVAL;
331 nfc_writel(host->nfc->hsmc_regs, BANK, ATMEL_HSMC_NFC_BANK1);
332 } else {
333 nfc_writel(host->nfc->hsmc_regs, BANK, ATMEL_HSMC_NFC_BANK0);
334 }
335
336 return 0;
337}
338
339static uint nfc_get_sram_off(struct atmel_nand_host *host)
340{
341 if (nfc_readl(host->nfc->hsmc_regs, BANK) & ATMEL_HSMC_NFC_BANK1)
342 return NFC_SRAM_BANK1_OFFSET;
343 else
344 return 0;
345}
346
347static dma_addr_t nfc_sram_phys(struct atmel_nand_host *host)
348{
349 if (nfc_readl(host->nfc->hsmc_regs, BANK) & ATMEL_HSMC_NFC_BANK1)
350 return host->nfc->sram_bank0_phys + NFC_SRAM_BANK1_OFFSET;
351 else
352 return host->nfc->sram_bank0_phys;
353}
354
355static int atmel_nand_dma_op(struct mtd_info *mtd, void *buf, int len,
356 int is_read)
357{
358 struct dma_device *dma_dev;
359 enum dma_ctrl_flags flags;
360 dma_addr_t dma_src_addr, dma_dst_addr, phys_addr;
361 struct dma_async_tx_descriptor *tx = NULL;
362 dma_cookie_t cookie;
363 struct nand_chip *chip = mtd_to_nand(mtd);
364 struct atmel_nand_host *host = nand_get_controller_data(chip);
365 void *p = buf;
366 int err = -EIO;
367 enum dma_data_direction dir = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
368 struct atmel_nfc *nfc = host->nfc;
369
370 if (buf >= high_memory)
371 goto err_buf;
372
373 dma_dev = host->dma_chan->device;
374
375 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
376
377 phys_addr = dma_map_single(dma_dev->dev, p, len, dir);
378 if (dma_mapping_error(dma_dev->dev, phys_addr)) {
379 dev_err(host->dev, "Failed to dma_map_single\n");
380 goto err_buf;
381 }
382
383 if (is_read) {
384 if (nfc && nfc->data_in_sram)
385 dma_src_addr = nfc_sram_phys(host) + (nfc->data_in_sram
386 - (nfc->sram_bank0 + nfc_get_sram_off(host)));
387 else
388 dma_src_addr = host->io_phys;
389
390 dma_dst_addr = phys_addr;
391 } else {
392 dma_src_addr = phys_addr;
393
394 if (nfc && nfc->write_by_sram)
395 dma_dst_addr = nfc_sram_phys(host);
396 else
397 dma_dst_addr = host->io_phys;
398 }
399
400 tx = dma_dev->device_prep_dma_memcpy(host->dma_chan, dma_dst_addr,
401 dma_src_addr, len, flags);
402 if (!tx) {
403 dev_err(host->dev, "Failed to prepare DMA memcpy\n");
404 goto err_dma;
405 }
406
407 init_completion(&host->comp);
408 tx->callback = dma_complete_func;
409 tx->callback_param = &host->comp;
410
411 cookie = tx->tx_submit(tx);
412 if (dma_submit_error(cookie)) {
413 dev_err(host->dev, "Failed to do DMA tx_submit\n");
414 goto err_dma;
415 }
416
417 dma_async_issue_pending(host->dma_chan);
418 wait_for_completion(&host->comp);
419
420 if (is_read && nfc && nfc->data_in_sram)
421 /* After read data from SRAM, need to increase the position */
422 nfc->data_in_sram += len;
423
424 err = 0;
425
426err_dma:
427 dma_unmap_single(dma_dev->dev, phys_addr, len, dir);
428err_buf:
429 if (err != 0)
430 dev_dbg(host->dev, "Fall back to CPU I/O\n");
431 return err;
432}
433
434static void atmel_read_buf(struct mtd_info *mtd, u8 *buf, int len)
435{
436 struct nand_chip *chip = mtd_to_nand(mtd);
437 struct atmel_nand_host *host = nand_get_controller_data(chip);
438
439 if (use_dma && len > mtd->oobsize)
440 /* only use DMA for bigger than oob size: better performances */
441 if (atmel_nand_dma_op(mtd, buf, len, 1) == 0)
442 return;
443
444 if (host->board.bus_width_16)
445 atmel_read_buf16(mtd, buf, len);
446 else
447 atmel_read_buf8(mtd, buf, len);
448}
449
450static void atmel_write_buf(struct mtd_info *mtd, const u8 *buf, int len)
451{
452 struct nand_chip *chip = mtd_to_nand(mtd);
453 struct atmel_nand_host *host = nand_get_controller_data(chip);
454
455 if (use_dma && len > mtd->oobsize)
456 /* only use DMA for bigger than oob size: better performances */
457 if (atmel_nand_dma_op(mtd, (void *)buf, len, 0) == 0)
458 return;
459
460 if (host->board.bus_width_16)
461 atmel_write_buf16(mtd, buf, len);
462 else
463 atmel_write_buf8(mtd, buf, len);
464}
465
466/*
467 * Return number of ecc bytes per sector according to sector size and
468 * correction capability
469 *
470 * Following table shows what at91 PMECC supported:
471 * Correction Capability Sector_512_bytes Sector_1024_bytes
472 * ===================== ================ =================
473 * 2-bits 4-bytes 4-bytes
474 * 4-bits 7-bytes 7-bytes
475 * 8-bits 13-bytes 14-bytes
476 * 12-bits 20-bytes 21-bytes
477 * 24-bits 39-bytes 42-bytes
478 * 32-bits 52-bytes 56-bytes
479 */
480static int pmecc_get_ecc_bytes(int cap, int sector_size)
481{
482 int m = 12 + sector_size / 512;
483 return (m * cap + 7) / 8;
484}
485
486static void pmecc_config_ecc_layout(struct nand_ecclayout *layout,
487 int oobsize, int ecc_len)
488{
489 int i;
490
491 layout->eccbytes = ecc_len;
492
493 /* ECC will occupy the last ecc_len bytes continuously */
494 for (i = 0; i < ecc_len; i++)
495 layout->eccpos[i] = oobsize - ecc_len + i;
496
497 layout->oobfree[0].offset = PMECC_OOB_RESERVED_BYTES;
498 layout->oobfree[0].length =
499 oobsize - ecc_len - layout->oobfree[0].offset;
500}
501
502static void __iomem *pmecc_get_alpha_to(struct atmel_nand_host *host)
503{
504 int table_size;
505
506 table_size = host->pmecc_sector_size == 512 ?
507 PMECC_LOOKUP_TABLE_SIZE_512 : PMECC_LOOKUP_TABLE_SIZE_1024;
508
509 return host->pmecc_rom_base + host->pmecc_lookup_table_offset +
510 table_size * sizeof(int16_t);
511}
512
513static int pmecc_data_alloc(struct atmel_nand_host *host)
514{
515 const int cap = host->pmecc_corr_cap;
516 int size;
517
518 size = (2 * cap + 1) * sizeof(int16_t);
519 host->pmecc_partial_syn = devm_kzalloc(host->dev, size, GFP_KERNEL);
520 host->pmecc_si = devm_kzalloc(host->dev, size, GFP_KERNEL);
521 host->pmecc_lmu = devm_kzalloc(host->dev,
522 (cap + 1) * sizeof(int16_t), GFP_KERNEL);
523 host->pmecc_smu = devm_kzalloc(host->dev,
524 (cap + 2) * size, GFP_KERNEL);
525
526 size = (cap + 1) * sizeof(int);
527 host->pmecc_mu = devm_kzalloc(host->dev, size, GFP_KERNEL);
528 host->pmecc_dmu = devm_kzalloc(host->dev, size, GFP_KERNEL);
529 host->pmecc_delta = devm_kzalloc(host->dev, size, GFP_KERNEL);
530
531 if (!host->pmecc_partial_syn ||
532 !host->pmecc_si ||
533 !host->pmecc_lmu ||
534 !host->pmecc_smu ||
535 !host->pmecc_mu ||
536 !host->pmecc_dmu ||
537 !host->pmecc_delta)
538 return -ENOMEM;
539
540 return 0;
541}
542
543static void pmecc_gen_syndrome(struct mtd_info *mtd, int sector)
544{
545 struct nand_chip *nand_chip = mtd_to_nand(mtd);
546 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
547 int i;
548 uint32_t value;
549
550 /* Fill odd syndromes */
551 for (i = 0; i < host->pmecc_corr_cap; i++) {
552 value = pmecc_readl_rem_relaxed(host->ecc, sector, i / 2);
553 if (i & 1)
554 value >>= 16;
555 value &= 0xffff;
556 host->pmecc_partial_syn[(2 * i) + 1] = (int16_t)value;
557 }
558}
559
560static void pmecc_substitute(struct mtd_info *mtd)
561{
562 struct nand_chip *nand_chip = mtd_to_nand(mtd);
563 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
564 int16_t __iomem *alpha_to = host->pmecc_alpha_to;
565 int16_t __iomem *index_of = host->pmecc_index_of;
566 int16_t *partial_syn = host->pmecc_partial_syn;
567 const int cap = host->pmecc_corr_cap;
568 int16_t *si;
569 int i, j;
570
571 /* si[] is a table that holds the current syndrome value,
572 * an element of that table belongs to the field
573 */
574 si = host->pmecc_si;
575
576 memset(&si[1], 0, sizeof(int16_t) * (2 * cap - 1));
577
578 /* Computation 2t syndromes based on S(x) */
579 /* Odd syndromes */
580 for (i = 1; i < 2 * cap; i += 2) {
581 for (j = 0; j < host->pmecc_degree; j++) {
582 if (partial_syn[i] & ((unsigned short)0x1 << j))
583 si[i] = readw_relaxed(alpha_to + i * j) ^ si[i];
584 }
585 }
586 /* Even syndrome = (Odd syndrome) ** 2 */
587 for (i = 2, j = 1; j <= cap; i = ++j << 1) {
588 if (si[j] == 0) {
589 si[i] = 0;
590 } else {
591 int16_t tmp;
592
593 tmp = readw_relaxed(index_of + si[j]);
594 tmp = (tmp * 2) % host->pmecc_cw_len;
595 si[i] = readw_relaxed(alpha_to + tmp);
596 }
597 }
598
599 return;
600}
601
602static void pmecc_get_sigma(struct mtd_info *mtd)
603{
604 struct nand_chip *nand_chip = mtd_to_nand(mtd);
605 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
606
607 int16_t *lmu = host->pmecc_lmu;
608 int16_t *si = host->pmecc_si;
609 int *mu = host->pmecc_mu;
610 int *dmu = host->pmecc_dmu; /* Discrepancy */
611 int *delta = host->pmecc_delta; /* Delta order */
612 int cw_len = host->pmecc_cw_len;
613 const int16_t cap = host->pmecc_corr_cap;
614 const int num = 2 * cap + 1;
615 int16_t __iomem *index_of = host->pmecc_index_of;
616 int16_t __iomem *alpha_to = host->pmecc_alpha_to;
617 int i, j, k;
618 uint32_t dmu_0_count, tmp;
619 int16_t *smu = host->pmecc_smu;
620
621 /* index of largest delta */
622 int ro;
623 int largest;
624 int diff;
625
626 dmu_0_count = 0;
627
628 /* First Row */
629
630 /* Mu */
631 mu[0] = -1;
632
633 memset(smu, 0, sizeof(int16_t) * num);
634 smu[0] = 1;
635
636 /* discrepancy set to 1 */
637 dmu[0] = 1;
638 /* polynom order set to 0 */
639 lmu[0] = 0;
640 delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
641
642 /* Second Row */
643
644 /* Mu */
645 mu[1] = 0;
646 /* Sigma(x) set to 1 */
647 memset(&smu[num], 0, sizeof(int16_t) * num);
648 smu[num] = 1;
649
650 /* discrepancy set to S1 */
651 dmu[1] = si[1];
652
653 /* polynom order set to 0 */
654 lmu[1] = 0;
655
656 delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
657
658 /* Init the Sigma(x) last row */
659 memset(&smu[(cap + 1) * num], 0, sizeof(int16_t) * num);
660
661 for (i = 1; i <= cap; i++) {
662 mu[i + 1] = i << 1;
663 /* Begin Computing Sigma (Mu+1) and L(mu) */
664 /* check if discrepancy is set to 0 */
665 if (dmu[i] == 0) {
666 dmu_0_count++;
667
668 tmp = ((cap - (lmu[i] >> 1) - 1) / 2);
669 if ((cap - (lmu[i] >> 1) - 1) & 0x1)
670 tmp += 2;
671 else
672 tmp += 1;
673
674 if (dmu_0_count == tmp) {
675 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
676 smu[(cap + 1) * num + j] =
677 smu[i * num + j];
678
679 lmu[cap + 1] = lmu[i];
680 return;
681 }
682
683 /* copy polynom */
684 for (j = 0; j <= lmu[i] >> 1; j++)
685 smu[(i + 1) * num + j] = smu[i * num + j];
686
687 /* copy previous polynom order to the next */
688 lmu[i + 1] = lmu[i];
689 } else {
690 ro = 0;
691 largest = -1;
692 /* find largest delta with dmu != 0 */
693 for (j = 0; j < i; j++) {
694 if ((dmu[j]) && (delta[j] > largest)) {
695 largest = delta[j];
696 ro = j;
697 }
698 }
699
700 /* compute difference */
701 diff = (mu[i] - mu[ro]);
702
703 /* Compute degree of the new smu polynomial */
704 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
705 lmu[i + 1] = lmu[i];
706 else
707 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
708
709 /* Init smu[i+1] with 0 */
710 for (k = 0; k < num; k++)
711 smu[(i + 1) * num + k] = 0;
712
713 /* Compute smu[i+1] */
714 for (k = 0; k <= lmu[ro] >> 1; k++) {
715 int16_t a, b, c;
716
717 if (!(smu[ro * num + k] && dmu[i]))
718 continue;
719 a = readw_relaxed(index_of + dmu[i]);
720 b = readw_relaxed(index_of + dmu[ro]);
721 c = readw_relaxed(index_of + smu[ro * num + k]);
722 tmp = a + (cw_len - b) + c;
723 a = readw_relaxed(alpha_to + tmp % cw_len);
724 smu[(i + 1) * num + (k + diff)] = a;
725 }
726
727 for (k = 0; k <= lmu[i] >> 1; k++)
728 smu[(i + 1) * num + k] ^= smu[i * num + k];
729 }
730
731 /* End Computing Sigma (Mu+1) and L(mu) */
732 /* In either case compute delta */
733 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
734
735 /* Do not compute discrepancy for the last iteration */
736 if (i >= cap)
737 continue;
738
739 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
740 tmp = 2 * (i - 1);
741 if (k == 0) {
742 dmu[i + 1] = si[tmp + 3];
743 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
744 int16_t a, b, c;
745 a = readw_relaxed(index_of +
746 smu[(i + 1) * num + k]);
747 b = si[2 * (i - 1) + 3 - k];
748 c = readw_relaxed(index_of + b);
749 tmp = a + c;
750 tmp %= cw_len;
751 dmu[i + 1] = readw_relaxed(alpha_to + tmp) ^
752 dmu[i + 1];
753 }
754 }
755 }
756
757 return;
758}
759
760static int pmecc_err_location(struct mtd_info *mtd)
761{
762 struct nand_chip *nand_chip = mtd_to_nand(mtd);
763 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
764 unsigned long end_time;
765 const int cap = host->pmecc_corr_cap;
766 const int num = 2 * cap + 1;
767 int sector_size = host->pmecc_sector_size;
768 int err_nbr = 0; /* number of error */
769 int roots_nbr; /* number of roots */
770 int i;
771 uint32_t val;
772 int16_t *smu = host->pmecc_smu;
773
774 pmerrloc_writel(host->pmerrloc_base, ELDIS, PMERRLOC_DISABLE);
775
776 for (i = 0; i <= host->pmecc_lmu[cap + 1] >> 1; i++) {
777 pmerrloc_writel_sigma_relaxed(host->pmerrloc_base, i,
778 smu[(cap + 1) * num + i]);
779 err_nbr++;
780 }
781
782 val = (err_nbr - 1) << 16;
783 if (sector_size == 1024)
784 val |= 1;
785
786 pmerrloc_writel(host->pmerrloc_base, ELCFG, val);
787 pmerrloc_writel(host->pmerrloc_base, ELEN,
788 sector_size * 8 + host->pmecc_degree * cap);
789
790 end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
791 while (!(pmerrloc_readl_relaxed(host->pmerrloc_base, ELISR)
792 & PMERRLOC_CALC_DONE)) {
793 if (unlikely(time_after(jiffies, end_time))) {
794 dev_err(host->dev, "PMECC: Timeout to calculate error location.\n");
795 return -1;
796 }
797 cpu_relax();
798 }
799
800 roots_nbr = (pmerrloc_readl_relaxed(host->pmerrloc_base, ELISR)
801 & PMERRLOC_ERR_NUM_MASK) >> 8;
802 /* Number of roots == degree of smu hence <= cap */
803 if (roots_nbr == host->pmecc_lmu[cap + 1] >> 1)
804 return err_nbr - 1;
805
806 /* Number of roots does not match the degree of smu
807 * unable to correct error */
808 return -1;
809}
810
811static void pmecc_correct_data(struct mtd_info *mtd, uint8_t *buf, uint8_t *ecc,
812 int sector_num, int extra_bytes, int err_nbr)
813{
814 struct nand_chip *nand_chip = mtd_to_nand(mtd);
815 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
816 int i = 0;
817 int byte_pos, bit_pos, sector_size, pos;
818 uint32_t tmp;
819 uint8_t err_byte;
820
821 sector_size = host->pmecc_sector_size;
822
823 while (err_nbr) {
824 tmp = pmerrloc_readl_el_relaxed(host->pmerrloc_el_base, i) - 1;
825 byte_pos = tmp / 8;
826 bit_pos = tmp % 8;
827
828 if (byte_pos >= (sector_size + extra_bytes))
829 BUG(); /* should never happen */
830
831 if (byte_pos < sector_size) {
832 err_byte = *(buf + byte_pos);
833 *(buf + byte_pos) ^= (1 << bit_pos);
834
835 pos = sector_num * host->pmecc_sector_size + byte_pos;
836 dev_dbg(host->dev, "Bit flip in data area, byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
837 pos, bit_pos, err_byte, *(buf + byte_pos));
838 } else {
839 /* Bit flip in OOB area */
840 tmp = sector_num * nand_chip->ecc.bytes
841 + (byte_pos - sector_size);
842 err_byte = ecc[tmp];
843 ecc[tmp] ^= (1 << bit_pos);
844
845 pos = tmp + nand_chip->ecc.layout->eccpos[0];
846 dev_dbg(host->dev, "Bit flip in OOB, oob_byte_pos: %d, bit_pos: %d, 0x%02x -> 0x%02x\n",
847 pos, bit_pos, err_byte, ecc[tmp]);
848 }
849
850 i++;
851 err_nbr--;
852 }
853
854 return;
855}
856
857static int pmecc_correction(struct mtd_info *mtd, u32 pmecc_stat, uint8_t *buf,
858 u8 *ecc)
859{
860 struct nand_chip *nand_chip = mtd_to_nand(mtd);
861 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
862 int i, err_nbr;
863 uint8_t *buf_pos;
864 int max_bitflips = 0;
865
866 /* If can correct bitfilps from erased page, do the normal check */
867 if (host->caps->pmecc_correct_erase_page)
868 goto normal_check;
869
870 for (i = 0; i < nand_chip->ecc.total; i++)
871 if (ecc[i] != 0xff)
872 goto normal_check;
873 /* Erased page, return OK */
874 return 0;
875
876normal_check:
877 for (i = 0; i < nand_chip->ecc.steps; i++) {
878 err_nbr = 0;
879 if (pmecc_stat & 0x1) {
880 buf_pos = buf + i * host->pmecc_sector_size;
881
882 pmecc_gen_syndrome(mtd, i);
883 pmecc_substitute(mtd);
884 pmecc_get_sigma(mtd);
885
886 err_nbr = pmecc_err_location(mtd);
887 if (err_nbr == -1) {
888 dev_err(host->dev, "PMECC: Too many errors\n");
889 mtd->ecc_stats.failed++;
890 return -EIO;
891 } else {
892 pmecc_correct_data(mtd, buf_pos, ecc, i,
893 nand_chip->ecc.bytes, err_nbr);
894 mtd->ecc_stats.corrected += err_nbr;
895 max_bitflips = max_t(int, max_bitflips, err_nbr);
896 }
897 }
898 pmecc_stat >>= 1;
899 }
900
901 return max_bitflips;
902}
903
904static void pmecc_enable(struct atmel_nand_host *host, int ecc_op)
905{
906 u32 val;
907
908 if (ecc_op != NAND_ECC_READ && ecc_op != NAND_ECC_WRITE) {
909 dev_err(host->dev, "atmel_nand: wrong pmecc operation type!");
910 return;
911 }
912
913 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_RST);
914 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
915 val = pmecc_readl_relaxed(host->ecc, CFG);
916
917 if (ecc_op == NAND_ECC_READ)
918 pmecc_writel(host->ecc, CFG, (val & ~PMECC_CFG_WRITE_OP)
919 | PMECC_CFG_AUTO_ENABLE);
920 else
921 pmecc_writel(host->ecc, CFG, (val | PMECC_CFG_WRITE_OP)
922 & ~PMECC_CFG_AUTO_ENABLE);
923
924 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_ENABLE);
925 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DATA);
926}
927
928static int atmel_nand_pmecc_read_page(struct mtd_info *mtd,
929 struct nand_chip *chip, uint8_t *buf, int oob_required, int page)
930{
931 struct atmel_nand_host *host = nand_get_controller_data(chip);
932 int eccsize = chip->ecc.size * chip->ecc.steps;
933 uint8_t *oob = chip->oob_poi;
934 uint32_t *eccpos = chip->ecc.layout->eccpos;
935 uint32_t stat;
936 unsigned long end_time;
937 int bitflips = 0;
938
939 if (!host->nfc || !host->nfc->use_nfc_sram)
940 pmecc_enable(host, NAND_ECC_READ);
941
942 chip->read_buf(mtd, buf, eccsize);
943 chip->read_buf(mtd, oob, mtd->oobsize);
944
945 end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
946 while ((pmecc_readl_relaxed(host->ecc, SR) & PMECC_SR_BUSY)) {
947 if (unlikely(time_after(jiffies, end_time))) {
948 dev_err(host->dev, "PMECC: Timeout to get error status.\n");
949 return -EIO;
950 }
951 cpu_relax();
952 }
953
954 stat = pmecc_readl_relaxed(host->ecc, ISR);
955 if (stat != 0) {
956 bitflips = pmecc_correction(mtd, stat, buf, &oob[eccpos[0]]);
957 if (bitflips < 0)
958 /* uncorrectable errors */
959 return 0;
960 }
961
962 return bitflips;
963}
964
965static int atmel_nand_pmecc_write_page(struct mtd_info *mtd,
966 struct nand_chip *chip, const uint8_t *buf, int oob_required,
967 int page)
968{
969 struct atmel_nand_host *host = nand_get_controller_data(chip);
970 uint32_t *eccpos = chip->ecc.layout->eccpos;
971 int i, j;
972 unsigned long end_time;
973
974 if (!host->nfc || !host->nfc->write_by_sram) {
975 pmecc_enable(host, NAND_ECC_WRITE);
976 chip->write_buf(mtd, (u8 *)buf, mtd->writesize);
977 }
978
979 end_time = jiffies + msecs_to_jiffies(PMECC_MAX_TIMEOUT_MS);
980 while ((pmecc_readl_relaxed(host->ecc, SR) & PMECC_SR_BUSY)) {
981 if (unlikely(time_after(jiffies, end_time))) {
982 dev_err(host->dev, "PMECC: Timeout to get ECC value.\n");
983 return -EIO;
984 }
985 cpu_relax();
986 }
987
988 for (i = 0; i < chip->ecc.steps; i++) {
989 for (j = 0; j < chip->ecc.bytes; j++) {
990 int pos;
991
992 pos = i * chip->ecc.bytes + j;
993 chip->oob_poi[eccpos[pos]] =
994 pmecc_readb_ecc_relaxed(host->ecc, i, j);
995 }
996 }
997 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
998
999 return 0;
1000}
1001
1002static void atmel_pmecc_core_init(struct mtd_info *mtd)
1003{
1004 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1005 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1006 uint32_t val = 0;
1007 struct nand_ecclayout *ecc_layout;
1008
1009 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_RST);
1010 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
1011
1012 switch (host->pmecc_corr_cap) {
1013 case 2:
1014 val = PMECC_CFG_BCH_ERR2;
1015 break;
1016 case 4:
1017 val = PMECC_CFG_BCH_ERR4;
1018 break;
1019 case 8:
1020 val = PMECC_CFG_BCH_ERR8;
1021 break;
1022 case 12:
1023 val = PMECC_CFG_BCH_ERR12;
1024 break;
1025 case 24:
1026 val = PMECC_CFG_BCH_ERR24;
1027 break;
1028 case 32:
1029 val = PMECC_CFG_BCH_ERR32;
1030 break;
1031 }
1032
1033 if (host->pmecc_sector_size == 512)
1034 val |= PMECC_CFG_SECTOR512;
1035 else if (host->pmecc_sector_size == 1024)
1036 val |= PMECC_CFG_SECTOR1024;
1037
1038 switch (nand_chip->ecc.steps) {
1039 case 1:
1040 val |= PMECC_CFG_PAGE_1SECTOR;
1041 break;
1042 case 2:
1043 val |= PMECC_CFG_PAGE_2SECTORS;
1044 break;
1045 case 4:
1046 val |= PMECC_CFG_PAGE_4SECTORS;
1047 break;
1048 case 8:
1049 val |= PMECC_CFG_PAGE_8SECTORS;
1050 break;
1051 }
1052
1053 val |= (PMECC_CFG_READ_OP | PMECC_CFG_SPARE_DISABLE
1054 | PMECC_CFG_AUTO_DISABLE);
1055 pmecc_writel(host->ecc, CFG, val);
1056
1057 ecc_layout = nand_chip->ecc.layout;
1058 pmecc_writel(host->ecc, SAREA, mtd->oobsize - 1);
1059 pmecc_writel(host->ecc, SADDR, ecc_layout->eccpos[0]);
1060 pmecc_writel(host->ecc, EADDR,
1061 ecc_layout->eccpos[ecc_layout->eccbytes - 1]);
1062 /* See datasheet about PMECC Clock Control Register */
1063 pmecc_writel(host->ecc, CLK, 2);
1064 pmecc_writel(host->ecc, IDR, 0xff);
1065 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_ENABLE);
1066}
1067
1068/*
1069 * Get minimum ecc requirements from NAND.
1070 * If pmecc-cap, pmecc-sector-size in DTS are not specified, this function
1071 * will set them according to minimum ecc requirement. Otherwise, use the
1072 * value in DTS file.
1073 * return 0 if success. otherwise return error code.
1074 */
1075static int pmecc_choose_ecc(struct atmel_nand_host *host,
1076 int *cap, int *sector_size)
1077{
1078 /* Get minimum ECC requirements */
1079 if (host->nand_chip.ecc_strength_ds) {
1080 *cap = host->nand_chip.ecc_strength_ds;
1081 *sector_size = host->nand_chip.ecc_step_ds;
1082 dev_info(host->dev, "minimum ECC: %d bits in %d bytes\n",
1083 *cap, *sector_size);
1084 } else {
1085 *cap = 2;
1086 *sector_size = 512;
1087 dev_info(host->dev, "can't detect min. ECC, assume 2 bits in 512 bytes\n");
1088 }
1089
1090 /* If device tree doesn't specify, use NAND's minimum ECC parameters */
1091 if (host->pmecc_corr_cap == 0) {
1092 if (*cap > host->caps->pmecc_max_correction)
1093 return -EINVAL;
1094
1095 /* use the most fitable ecc bits (the near bigger one ) */
1096 if (*cap <= 2)
1097 host->pmecc_corr_cap = 2;
1098 else if (*cap <= 4)
1099 host->pmecc_corr_cap = 4;
1100 else if (*cap <= 8)
1101 host->pmecc_corr_cap = 8;
1102 else if (*cap <= 12)
1103 host->pmecc_corr_cap = 12;
1104 else if (*cap <= 24)
1105 host->pmecc_corr_cap = 24;
1106 else if (*cap <= 32)
1107 host->pmecc_corr_cap = 32;
1108 else
1109 return -EINVAL;
1110 }
1111 if (host->pmecc_sector_size == 0) {
1112 /* use the most fitable sector size (the near smaller one ) */
1113 if (*sector_size >= 1024)
1114 host->pmecc_sector_size = 1024;
1115 else if (*sector_size >= 512)
1116 host->pmecc_sector_size = 512;
1117 else
1118 return -EINVAL;
1119 }
1120 return 0;
1121}
1122
1123static inline int deg(unsigned int poly)
1124{
1125 /* polynomial degree is the most-significant bit index */
1126 return fls(poly) - 1;
1127}
1128
1129static int build_gf_tables(int mm, unsigned int poly,
1130 int16_t *index_of, int16_t *alpha_to)
1131{
1132 unsigned int i, x = 1;
1133 const unsigned int k = 1 << deg(poly);
1134 unsigned int nn = (1 << mm) - 1;
1135
1136 /* primitive polynomial must be of degree m */
1137 if (k != (1u << mm))
1138 return -EINVAL;
1139
1140 for (i = 0; i < nn; i++) {
1141 alpha_to[i] = x;
1142 index_of[x] = i;
1143 if (i && (x == 1))
1144 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
1145 return -EINVAL;
1146 x <<= 1;
1147 if (x & k)
1148 x ^= poly;
1149 }
1150 alpha_to[nn] = 1;
1151 index_of[0] = 0;
1152
1153 return 0;
1154}
1155
1156static uint16_t *create_lookup_table(struct device *dev, int sector_size)
1157{
1158 int degree = (sector_size == 512) ?
1159 PMECC_GF_DIMENSION_13 :
1160 PMECC_GF_DIMENSION_14;
1161 unsigned int poly = (sector_size == 512) ?
1162 PMECC_GF_13_PRIMITIVE_POLY :
1163 PMECC_GF_14_PRIMITIVE_POLY;
1164 int table_size = (sector_size == 512) ?
1165 PMECC_LOOKUP_TABLE_SIZE_512 :
1166 PMECC_LOOKUP_TABLE_SIZE_1024;
1167
1168 int16_t *addr = devm_kzalloc(dev, 2 * table_size * sizeof(uint16_t),
1169 GFP_KERNEL);
1170 if (addr && build_gf_tables(degree, poly, addr, addr + table_size))
1171 return NULL;
1172
1173 return addr;
1174}
1175
1176static int atmel_pmecc_nand_init_params(struct platform_device *pdev,
1177 struct atmel_nand_host *host)
1178{
1179 struct nand_chip *nand_chip = &host->nand_chip;
1180 struct mtd_info *mtd = nand_to_mtd(nand_chip);
1181 struct resource *regs, *regs_pmerr, *regs_rom;
1182 uint16_t *galois_table;
1183 int cap, sector_size, err_no;
1184
1185 err_no = pmecc_choose_ecc(host, &cap, §or_size);
1186 if (err_no) {
1187 dev_err(host->dev, "The NAND flash's ECC requirement are not support!");
1188 return err_no;
1189 }
1190
1191 if (cap > host->pmecc_corr_cap ||
1192 sector_size != host->pmecc_sector_size)
1193 dev_info(host->dev, "WARNING: Be Caution! Using different PMECC parameters from Nand ONFI ECC reqirement.\n");
1194
1195 cap = host->pmecc_corr_cap;
1196 sector_size = host->pmecc_sector_size;
1197 host->pmecc_lookup_table_offset = (sector_size == 512) ?
1198 host->pmecc_lookup_table_offset_512 :
1199 host->pmecc_lookup_table_offset_1024;
1200
1201 dev_info(host->dev, "Initialize PMECC params, cap: %d, sector: %d\n",
1202 cap, sector_size);
1203
1204 regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1205 if (!regs) {
1206 dev_warn(host->dev,
1207 "Can't get I/O resource regs for PMECC controller, rolling back on software ECC\n");
1208 nand_chip->ecc.mode = NAND_ECC_SOFT;
1209 return 0;
1210 }
1211
1212 host->ecc = devm_ioremap_resource(&pdev->dev, regs);
1213 if (IS_ERR(host->ecc)) {
1214 err_no = PTR_ERR(host->ecc);
1215 goto err;
1216 }
1217
1218 regs_pmerr = platform_get_resource(pdev, IORESOURCE_MEM, 2);
1219 host->pmerrloc_base = devm_ioremap_resource(&pdev->dev, regs_pmerr);
1220 if (IS_ERR(host->pmerrloc_base)) {
1221 err_no = PTR_ERR(host->pmerrloc_base);
1222 goto err;
1223 }
1224 host->pmerrloc_el_base = host->pmerrloc_base + ATMEL_PMERRLOC_SIGMAx +
1225 (host->caps->pmecc_max_correction + 1) * 4;
1226
1227 if (!host->has_no_lookup_table) {
1228 regs_rom = platform_get_resource(pdev, IORESOURCE_MEM, 3);
1229 host->pmecc_rom_base = devm_ioremap_resource(&pdev->dev,
1230 regs_rom);
1231 if (IS_ERR(host->pmecc_rom_base)) {
1232 dev_err(host->dev, "Can not get I/O resource for ROM, will build a lookup table in runtime!\n");
1233 host->has_no_lookup_table = true;
1234 }
1235 }
1236
1237 if (host->has_no_lookup_table) {
1238 /* Build the look-up table in runtime */
1239 galois_table = create_lookup_table(host->dev, sector_size);
1240 if (!galois_table) {
1241 dev_err(host->dev, "Failed to build a lookup table in runtime!\n");
1242 err_no = -EINVAL;
1243 goto err;
1244 }
1245
1246 host->pmecc_rom_base = (void __iomem *)galois_table;
1247 host->pmecc_lookup_table_offset = 0;
1248 }
1249
1250 nand_chip->ecc.size = sector_size;
1251
1252 /* set ECC page size and oob layout */
1253 switch (mtd->writesize) {
1254 case 512:
1255 case 1024:
1256 case 2048:
1257 case 4096:
1258 case 8192:
1259 if (sector_size > mtd->writesize) {
1260 dev_err(host->dev, "pmecc sector size is bigger than the page size!\n");
1261 err_no = -EINVAL;
1262 goto err;
1263 }
1264
1265 host->pmecc_degree = (sector_size == 512) ?
1266 PMECC_GF_DIMENSION_13 : PMECC_GF_DIMENSION_14;
1267 host->pmecc_cw_len = (1 << host->pmecc_degree) - 1;
1268 host->pmecc_alpha_to = pmecc_get_alpha_to(host);
1269 host->pmecc_index_of = host->pmecc_rom_base +
1270 host->pmecc_lookup_table_offset;
1271
1272 nand_chip->ecc.strength = cap;
1273 nand_chip->ecc.bytes = pmecc_get_ecc_bytes(cap, sector_size);
1274 nand_chip->ecc.steps = mtd->writesize / sector_size;
1275 nand_chip->ecc.total = nand_chip->ecc.bytes *
1276 nand_chip->ecc.steps;
1277 if (nand_chip->ecc.total >
1278 mtd->oobsize - PMECC_OOB_RESERVED_BYTES) {
1279 dev_err(host->dev, "No room for ECC bytes\n");
1280 err_no = -EINVAL;
1281 goto err;
1282 }
1283 pmecc_config_ecc_layout(&atmel_pmecc_oobinfo,
1284 mtd->oobsize,
1285 nand_chip->ecc.total);
1286
1287 nand_chip->ecc.layout = &atmel_pmecc_oobinfo;
1288 break;
1289 default:
1290 dev_warn(host->dev,
1291 "Unsupported page size for PMECC, use Software ECC\n");
1292 /* page size not handled by HW ECC */
1293 /* switching back to soft ECC */
1294 nand_chip->ecc.mode = NAND_ECC_SOFT;
1295 return 0;
1296 }
1297
1298 /* Allocate data for PMECC computation */
1299 err_no = pmecc_data_alloc(host);
1300 if (err_no) {
1301 dev_err(host->dev,
1302 "Cannot allocate memory for PMECC computation!\n");
1303 goto err;
1304 }
1305
1306 nand_chip->options |= NAND_NO_SUBPAGE_WRITE;
1307 nand_chip->ecc.read_page = atmel_nand_pmecc_read_page;
1308 nand_chip->ecc.write_page = atmel_nand_pmecc_write_page;
1309
1310 atmel_pmecc_core_init(mtd);
1311
1312 return 0;
1313
1314err:
1315 return err_no;
1316}
1317
1318/*
1319 * Calculate HW ECC
1320 *
1321 * function called after a write
1322 *
1323 * mtd: MTD block structure
1324 * dat: raw data (unused)
1325 * ecc_code: buffer for ECC
1326 */
1327static int atmel_nand_calculate(struct mtd_info *mtd,
1328 const u_char *dat, unsigned char *ecc_code)
1329{
1330 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1331 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1332 unsigned int ecc_value;
1333
1334 /* get the first 2 ECC bytes */
1335 ecc_value = ecc_readl(host->ecc, PR);
1336
1337 ecc_code[0] = ecc_value & 0xFF;
1338 ecc_code[1] = (ecc_value >> 8) & 0xFF;
1339
1340 /* get the last 2 ECC bytes */
1341 ecc_value = ecc_readl(host->ecc, NPR) & ATMEL_ECC_NPARITY;
1342
1343 ecc_code[2] = ecc_value & 0xFF;
1344 ecc_code[3] = (ecc_value >> 8) & 0xFF;
1345
1346 return 0;
1347}
1348
1349/*
1350 * HW ECC read page function
1351 *
1352 * mtd: mtd info structure
1353 * chip: nand chip info structure
1354 * buf: buffer to store read data
1355 * oob_required: caller expects OOB data read to chip->oob_poi
1356 */
1357static int atmel_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1358 uint8_t *buf, int oob_required, int page)
1359{
1360 int eccsize = chip->ecc.size;
1361 int eccbytes = chip->ecc.bytes;
1362 uint32_t *eccpos = chip->ecc.layout->eccpos;
1363 uint8_t *p = buf;
1364 uint8_t *oob = chip->oob_poi;
1365 uint8_t *ecc_pos;
1366 int stat;
1367 unsigned int max_bitflips = 0;
1368
1369 /*
1370 * Errata: ALE is incorrectly wired up to the ECC controller
1371 * on the AP7000, so it will include the address cycles in the
1372 * ECC calculation.
1373 *
1374 * Workaround: Reset the parity registers before reading the
1375 * actual data.
1376 */
1377 struct atmel_nand_host *host = nand_get_controller_data(chip);
1378 if (host->board.need_reset_workaround)
1379 ecc_writel(host->ecc, CR, ATMEL_ECC_RST);
1380
1381 /* read the page */
1382 chip->read_buf(mtd, p, eccsize);
1383
1384 /* move to ECC position if needed */
1385 if (eccpos[0] != 0) {
1386 /* This only works on large pages
1387 * because the ECC controller waits for
1388 * NAND_CMD_RNDOUTSTART after the
1389 * NAND_CMD_RNDOUT.
1390 * anyway, for small pages, the eccpos[0] == 0
1391 */
1392 chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
1393 mtd->writesize + eccpos[0], -1);
1394 }
1395
1396 /* the ECC controller needs to read the ECC just after the data */
1397 ecc_pos = oob + eccpos[0];
1398 chip->read_buf(mtd, ecc_pos, eccbytes);
1399
1400 /* check if there's an error */
1401 stat = chip->ecc.correct(mtd, p, oob, NULL);
1402
1403 if (stat < 0) {
1404 mtd->ecc_stats.failed++;
1405 } else {
1406 mtd->ecc_stats.corrected += stat;
1407 max_bitflips = max_t(unsigned int, max_bitflips, stat);
1408 }
1409
1410 /* get back to oob start (end of page) */
1411 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
1412
1413 /* read the oob */
1414 chip->read_buf(mtd, oob, mtd->oobsize);
1415
1416 return max_bitflips;
1417}
1418
1419/*
1420 * HW ECC Correction
1421 *
1422 * function called after a read
1423 *
1424 * mtd: MTD block structure
1425 * dat: raw data read from the chip
1426 * read_ecc: ECC from the chip (unused)
1427 * isnull: unused
1428 *
1429 * Detect and correct a 1 bit error for a page
1430 */
1431static int atmel_nand_correct(struct mtd_info *mtd, u_char *dat,
1432 u_char *read_ecc, u_char *isnull)
1433{
1434 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1435 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1436 unsigned int ecc_status;
1437 unsigned int ecc_word, ecc_bit;
1438
1439 /* get the status from the Status Register */
1440 ecc_status = ecc_readl(host->ecc, SR);
1441
1442 /* if there's no error */
1443 if (likely(!(ecc_status & ATMEL_ECC_RECERR)))
1444 return 0;
1445
1446 /* get error bit offset (4 bits) */
1447 ecc_bit = ecc_readl(host->ecc, PR) & ATMEL_ECC_BITADDR;
1448 /* get word address (12 bits) */
1449 ecc_word = ecc_readl(host->ecc, PR) & ATMEL_ECC_WORDADDR;
1450 ecc_word >>= 4;
1451
1452 /* if there are multiple errors */
1453 if (ecc_status & ATMEL_ECC_MULERR) {
1454 /* check if it is a freshly erased block
1455 * (filled with 0xff) */
1456 if ((ecc_bit == ATMEL_ECC_BITADDR)
1457 && (ecc_word == (ATMEL_ECC_WORDADDR >> 4))) {
1458 /* the block has just been erased, return OK */
1459 return 0;
1460 }
1461 /* it doesn't seems to be a freshly
1462 * erased block.
1463 * We can't correct so many errors */
1464 dev_dbg(host->dev, "atmel_nand : multiple errors detected."
1465 " Unable to correct.\n");
1466 return -EBADMSG;
1467 }
1468
1469 /* if there's a single bit error : we can correct it */
1470 if (ecc_status & ATMEL_ECC_ECCERR) {
1471 /* there's nothing much to do here.
1472 * the bit error is on the ECC itself.
1473 */
1474 dev_dbg(host->dev, "atmel_nand : one bit error on ECC code."
1475 " Nothing to correct\n");
1476 return 0;
1477 }
1478
1479 dev_dbg(host->dev, "atmel_nand : one bit error on data."
1480 " (word offset in the page :"
1481 " 0x%x bit offset : 0x%x)\n",
1482 ecc_word, ecc_bit);
1483 /* correct the error */
1484 if (nand_chip->options & NAND_BUSWIDTH_16) {
1485 /* 16 bits words */
1486 ((unsigned short *) dat)[ecc_word] ^= (1 << ecc_bit);
1487 } else {
1488 /* 8 bits words */
1489 dat[ecc_word] ^= (1 << ecc_bit);
1490 }
1491 dev_dbg(host->dev, "atmel_nand : error corrected\n");
1492 return 1;
1493}
1494
1495/*
1496 * Enable HW ECC : unused on most chips
1497 */
1498static void atmel_nand_hwctl(struct mtd_info *mtd, int mode)
1499{
1500 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1501 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1502
1503 if (host->board.need_reset_workaround)
1504 ecc_writel(host->ecc, CR, ATMEL_ECC_RST);
1505}
1506
1507static int atmel_of_init_port(struct atmel_nand_host *host,
1508 struct device_node *np)
1509{
1510 u32 val;
1511 u32 offset[2];
1512 int ecc_mode;
1513 struct atmel_nand_data *board = &host->board;
1514 enum of_gpio_flags flags = 0;
1515
1516 host->caps = (struct atmel_nand_caps *)
1517 of_device_get_match_data(host->dev);
1518
1519 if (of_property_read_u32(np, "atmel,nand-addr-offset", &val) == 0) {
1520 if (val >= 32) {
1521 dev_err(host->dev, "invalid addr-offset %u\n", val);
1522 return -EINVAL;
1523 }
1524 board->ale = val;
1525 }
1526
1527 if (of_property_read_u32(np, "atmel,nand-cmd-offset", &val) == 0) {
1528 if (val >= 32) {
1529 dev_err(host->dev, "invalid cmd-offset %u\n", val);
1530 return -EINVAL;
1531 }
1532 board->cle = val;
1533 }
1534
1535 ecc_mode = of_get_nand_ecc_mode(np);
1536
1537 board->ecc_mode = ecc_mode < 0 ? NAND_ECC_SOFT : ecc_mode;
1538
1539 board->on_flash_bbt = of_get_nand_on_flash_bbt(np);
1540
1541 board->has_dma = of_property_read_bool(np, "atmel,nand-has-dma");
1542
1543 if (of_get_nand_bus_width(np) == 16)
1544 board->bus_width_16 = 1;
1545
1546 board->rdy_pin = of_get_gpio_flags(np, 0, &flags);
1547 board->rdy_pin_active_low = (flags == OF_GPIO_ACTIVE_LOW);
1548
1549 board->enable_pin = of_get_gpio(np, 1);
1550 board->det_pin = of_get_gpio(np, 2);
1551
1552 host->has_pmecc = of_property_read_bool(np, "atmel,has-pmecc");
1553
1554 /* load the nfc driver if there is */
1555 of_platform_populate(np, NULL, NULL, host->dev);
1556
1557 if (!(board->ecc_mode == NAND_ECC_HW) || !host->has_pmecc)
1558 return 0; /* Not using PMECC */
1559
1560 /* use PMECC, get correction capability, sector size and lookup
1561 * table offset.
1562 * If correction bits and sector size are not specified, then find
1563 * them from NAND ONFI parameters.
1564 */
1565 if (of_property_read_u32(np, "atmel,pmecc-cap", &val) == 0) {
1566 if (val > host->caps->pmecc_max_correction) {
1567 dev_err(host->dev,
1568 "Required ECC strength too high: %u max %u\n",
1569 val, host->caps->pmecc_max_correction);
1570 return -EINVAL;
1571 }
1572 if ((val != 2) && (val != 4) && (val != 8) &&
1573 (val != 12) && (val != 24) && (val != 32)) {
1574 dev_err(host->dev,
1575 "Required ECC strength not supported: %u\n",
1576 val);
1577 return -EINVAL;
1578 }
1579 host->pmecc_corr_cap = (u8)val;
1580 }
1581
1582 if (of_property_read_u32(np, "atmel,pmecc-sector-size", &val) == 0) {
1583 if ((val != 512) && (val != 1024)) {
1584 dev_err(host->dev,
1585 "Required ECC sector size not supported: %u\n",
1586 val);
1587 return -EINVAL;
1588 }
1589 host->pmecc_sector_size = (u16)val;
1590 }
1591
1592 if (of_property_read_u32_array(np, "atmel,pmecc-lookup-table-offset",
1593 offset, 2) != 0) {
1594 dev_err(host->dev, "Cannot get PMECC lookup table offset, will build a lookup table in runtime.\n");
1595 host->has_no_lookup_table = true;
1596 /* Will build a lookup table and initialize the offset later */
1597 return 0;
1598 }
1599 if (!offset[0] && !offset[1]) {
1600 dev_err(host->dev, "Invalid PMECC lookup table offset\n");
1601 return -EINVAL;
1602 }
1603 host->pmecc_lookup_table_offset_512 = offset[0];
1604 host->pmecc_lookup_table_offset_1024 = offset[1];
1605
1606 return 0;
1607}
1608
1609static int atmel_hw_nand_init_params(struct platform_device *pdev,
1610 struct atmel_nand_host *host)
1611{
1612 struct nand_chip *nand_chip = &host->nand_chip;
1613 struct mtd_info *mtd = nand_to_mtd(nand_chip);
1614 struct resource *regs;
1615
1616 regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1617 if (!regs) {
1618 dev_err(host->dev,
1619 "Can't get I/O resource regs, use software ECC\n");
1620 nand_chip->ecc.mode = NAND_ECC_SOFT;
1621 return 0;
1622 }
1623
1624 host->ecc = devm_ioremap_resource(&pdev->dev, regs);
1625 if (IS_ERR(host->ecc))
1626 return PTR_ERR(host->ecc);
1627
1628 /* ECC is calculated for the whole page (1 step) */
1629 nand_chip->ecc.size = mtd->writesize;
1630
1631 /* set ECC page size and oob layout */
1632 switch (mtd->writesize) {
1633 case 512:
1634 nand_chip->ecc.layout = &atmel_oobinfo_small;
1635 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_528);
1636 break;
1637 case 1024:
1638 nand_chip->ecc.layout = &atmel_oobinfo_large;
1639 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_1056);
1640 break;
1641 case 2048:
1642 nand_chip->ecc.layout = &atmel_oobinfo_large;
1643 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_2112);
1644 break;
1645 case 4096:
1646 nand_chip->ecc.layout = &atmel_oobinfo_large;
1647 ecc_writel(host->ecc, MR, ATMEL_ECC_PAGESIZE_4224);
1648 break;
1649 default:
1650 /* page size not handled by HW ECC */
1651 /* switching back to soft ECC */
1652 nand_chip->ecc.mode = NAND_ECC_SOFT;
1653 return 0;
1654 }
1655
1656 /* set up for HW ECC */
1657 nand_chip->ecc.calculate = atmel_nand_calculate;
1658 nand_chip->ecc.correct = atmel_nand_correct;
1659 nand_chip->ecc.hwctl = atmel_nand_hwctl;
1660 nand_chip->ecc.read_page = atmel_nand_read_page;
1661 nand_chip->ecc.bytes = 4;
1662 nand_chip->ecc.strength = 1;
1663
1664 return 0;
1665}
1666
1667static inline u32 nfc_read_status(struct atmel_nand_host *host)
1668{
1669 u32 err_flags = NFC_SR_DTOE | NFC_SR_UNDEF | NFC_SR_AWB | NFC_SR_ASE;
1670 u32 nfc_status = nfc_readl(host->nfc->hsmc_regs, SR);
1671
1672 if (unlikely(nfc_status & err_flags)) {
1673 if (nfc_status & NFC_SR_DTOE)
1674 dev_err(host->dev, "NFC: Waiting Nand R/B Timeout Error\n");
1675 else if (nfc_status & NFC_SR_UNDEF)
1676 dev_err(host->dev, "NFC: Access Undefined Area Error\n");
1677 else if (nfc_status & NFC_SR_AWB)
1678 dev_err(host->dev, "NFC: Access memory While NFC is busy\n");
1679 else if (nfc_status & NFC_SR_ASE)
1680 dev_err(host->dev, "NFC: Access memory Size Error\n");
1681 }
1682
1683 return nfc_status;
1684}
1685
1686/* SMC interrupt service routine */
1687static irqreturn_t hsmc_interrupt(int irq, void *dev_id)
1688{
1689 struct atmel_nand_host *host = dev_id;
1690 u32 status, mask, pending;
1691 irqreturn_t ret = IRQ_NONE;
1692
1693 status = nfc_read_status(host);
1694 mask = nfc_readl(host->nfc->hsmc_regs, IMR);
1695 pending = status & mask;
1696
1697 if (pending & NFC_SR_XFR_DONE) {
1698 complete(&host->nfc->comp_xfer_done);
1699 nfc_writel(host->nfc->hsmc_regs, IDR, NFC_SR_XFR_DONE);
1700 ret = IRQ_HANDLED;
1701 }
1702 if (pending & host->nfc->caps->rb_mask) {
1703 complete(&host->nfc->comp_ready);
1704 nfc_writel(host->nfc->hsmc_regs, IDR, host->nfc->caps->rb_mask);
1705 ret = IRQ_HANDLED;
1706 }
1707 if (pending & NFC_SR_CMD_DONE) {
1708 complete(&host->nfc->comp_cmd_done);
1709 nfc_writel(host->nfc->hsmc_regs, IDR, NFC_SR_CMD_DONE);
1710 ret = IRQ_HANDLED;
1711 }
1712
1713 return ret;
1714}
1715
1716/* NFC(Nand Flash Controller) related functions */
1717static void nfc_prepare_interrupt(struct atmel_nand_host *host, u32 flag)
1718{
1719 if (flag & NFC_SR_XFR_DONE)
1720 init_completion(&host->nfc->comp_xfer_done);
1721
1722 if (flag & host->nfc->caps->rb_mask)
1723 init_completion(&host->nfc->comp_ready);
1724
1725 if (flag & NFC_SR_CMD_DONE)
1726 init_completion(&host->nfc->comp_cmd_done);
1727
1728 /* Enable interrupt that need to wait for */
1729 nfc_writel(host->nfc->hsmc_regs, IER, flag);
1730}
1731
1732static int nfc_wait_interrupt(struct atmel_nand_host *host, u32 flag)
1733{
1734 int i, index = 0;
1735 struct completion *comp[3]; /* Support 3 interrupt completion */
1736
1737 if (flag & NFC_SR_XFR_DONE)
1738 comp[index++] = &host->nfc->comp_xfer_done;
1739
1740 if (flag & host->nfc->caps->rb_mask)
1741 comp[index++] = &host->nfc->comp_ready;
1742
1743 if (flag & NFC_SR_CMD_DONE)
1744 comp[index++] = &host->nfc->comp_cmd_done;
1745
1746 if (index == 0) {
1747 dev_err(host->dev, "Unknown interrupt flag: 0x%08x\n", flag);
1748 return -EINVAL;
1749 }
1750
1751 for (i = 0; i < index; i++) {
1752 if (wait_for_completion_timeout(comp[i],
1753 msecs_to_jiffies(NFC_TIME_OUT_MS)))
1754 continue; /* wait for next completion */
1755 else
1756 goto err_timeout;
1757 }
1758
1759 return 0;
1760
1761err_timeout:
1762 dev_err(host->dev, "Time out to wait for interrupt: 0x%08x\n", flag);
1763 /* Disable the interrupt as it is not handled by interrupt handler */
1764 nfc_writel(host->nfc->hsmc_regs, IDR, flag);
1765 return -ETIMEDOUT;
1766}
1767
1768static int nfc_send_command(struct atmel_nand_host *host,
1769 unsigned int cmd, unsigned int addr, unsigned char cycle0)
1770{
1771 unsigned long timeout;
1772 u32 flag = NFC_SR_CMD_DONE;
1773 flag |= cmd & NFCADDR_CMD_DATAEN ? NFC_SR_XFR_DONE : 0;
1774
1775 dev_dbg(host->dev,
1776 "nfc_cmd: 0x%08x, addr1234: 0x%08x, cycle0: 0x%02x\n",
1777 cmd, addr, cycle0);
1778
1779 timeout = jiffies + msecs_to_jiffies(NFC_TIME_OUT_MS);
1780 while (nfc_readl(host->nfc->hsmc_regs, SR) & NFC_SR_BUSY) {
1781 if (time_after(jiffies, timeout)) {
1782 dev_err(host->dev,
1783 "Time out to wait for NFC ready!\n");
1784 return -ETIMEDOUT;
1785 }
1786 }
1787
1788 nfc_prepare_interrupt(host, flag);
1789 nfc_writel(host->nfc->hsmc_regs, CYCLE0, cycle0);
1790 nfc_cmd_addr1234_writel(cmd, addr, host->nfc->base_cmd_regs);
1791 return nfc_wait_interrupt(host, flag);
1792}
1793
1794static int nfc_device_ready(struct mtd_info *mtd)
1795{
1796 u32 status, mask;
1797 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1798 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1799
1800 status = nfc_read_status(host);
1801 mask = nfc_readl(host->nfc->hsmc_regs, IMR);
1802
1803 /* The mask should be 0. If not we may lost interrupts */
1804 if (unlikely(mask & status))
1805 dev_err(host->dev, "Lost the interrupt flags: 0x%08x\n",
1806 mask & status);
1807
1808 return status & host->nfc->caps->rb_mask;
1809}
1810
1811static void nfc_select_chip(struct mtd_info *mtd, int chip)
1812{
1813 struct nand_chip *nand_chip = mtd_to_nand(mtd);
1814 struct atmel_nand_host *host = nand_get_controller_data(nand_chip);
1815
1816 if (chip == -1)
1817 nfc_writel(host->nfc->hsmc_regs, CTRL, NFC_CTRL_DISABLE);
1818 else
1819 nfc_writel(host->nfc->hsmc_regs, CTRL, NFC_CTRL_ENABLE);
1820}
1821
1822static int nfc_make_addr(struct mtd_info *mtd, int command, int column,
1823 int page_addr, unsigned int *addr1234, unsigned int *cycle0)
1824{
1825 struct nand_chip *chip = mtd_to_nand(mtd);
1826
1827 int acycle = 0;
1828 unsigned char addr_bytes[8];
1829 int index = 0, bit_shift;
1830
1831 BUG_ON(addr1234 == NULL || cycle0 == NULL);
1832
1833 *cycle0 = 0;
1834 *addr1234 = 0;
1835
1836 if (column != -1) {
1837 if (chip->options & NAND_BUSWIDTH_16 &&
1838 !nand_opcode_8bits(command))
1839 column >>= 1;
1840 addr_bytes[acycle++] = column & 0xff;
1841 if (mtd->writesize > 512)
1842 addr_bytes[acycle++] = (column >> 8) & 0xff;
1843 }
1844
1845 if (page_addr != -1) {
1846 addr_bytes[acycle++] = page_addr & 0xff;
1847 addr_bytes[acycle++] = (page_addr >> 8) & 0xff;
1848 if (chip->chipsize > (128 << 20))
1849 addr_bytes[acycle++] = (page_addr >> 16) & 0xff;
1850 }
1851
1852 if (acycle > 4)
1853 *cycle0 = addr_bytes[index++];
1854
1855 for (bit_shift = 0; index < acycle; bit_shift += 8)
1856 *addr1234 += addr_bytes[index++] << bit_shift;
1857
1858 /* return acycle in cmd register */
1859 return acycle << NFCADDR_CMD_ACYCLE_BIT_POS;
1860}
1861
1862static void nfc_nand_command(struct mtd_info *mtd, unsigned int command,
1863 int column, int page_addr)
1864{
1865 struct nand_chip *chip = mtd_to_nand(mtd);
1866 struct atmel_nand_host *host = nand_get_controller_data(chip);
1867 unsigned long timeout;
1868 unsigned int nfc_addr_cmd = 0;
1869
1870 unsigned int cmd1 = command << NFCADDR_CMD_CMD1_BIT_POS;
1871
1872 /* Set default settings: no cmd2, no addr cycle. read from nand */
1873 unsigned int cmd2 = 0;
1874 unsigned int vcmd2 = 0;
1875 int acycle = NFCADDR_CMD_ACYCLE_NONE;
1876 int csid = NFCADDR_CMD_CSID_3;
1877 int dataen = NFCADDR_CMD_DATADIS;
1878 int nfcwr = NFCADDR_CMD_NFCRD;
1879 unsigned int addr1234 = 0;
1880 unsigned int cycle0 = 0;
1881 bool do_addr = true;
1882 host->nfc->data_in_sram = NULL;
1883
1884 dev_dbg(host->dev, "%s: cmd = 0x%02x, col = 0x%08x, page = 0x%08x\n",
1885 __func__, command, column, page_addr);
1886
1887 switch (command) {
1888 case NAND_CMD_RESET:
1889 nfc_addr_cmd = cmd1 | acycle | csid | dataen | nfcwr;
1890 nfc_send_command(host, nfc_addr_cmd, addr1234, cycle0);
1891 udelay(chip->chip_delay);
1892
1893 nfc_nand_command(mtd, NAND_CMD_STATUS, -1, -1);
1894 timeout = jiffies + msecs_to_jiffies(NFC_TIME_OUT_MS);
1895 while (!(chip->read_byte(mtd) & NAND_STATUS_READY)) {
1896 if (time_after(jiffies, timeout)) {
1897 dev_err(host->dev,
1898 "Time out to wait status ready!\n");
1899 break;
1900 }
1901 }
1902 return;
1903 case NAND_CMD_STATUS:
1904 do_addr = false;
1905 break;
1906 case NAND_CMD_PARAM:
1907 case NAND_CMD_READID:
1908 do_addr = false;
1909 acycle = NFCADDR_CMD_ACYCLE_1;
1910 if (column != -1)
1911 addr1234 = column;
1912 break;
1913 case NAND_CMD_RNDOUT:
1914 cmd2 = NAND_CMD_RNDOUTSTART << NFCADDR_CMD_CMD2_BIT_POS;
1915 vcmd2 = NFCADDR_CMD_VCMD2;
1916 break;
1917 case NAND_CMD_READ0:
1918 case NAND_CMD_READOOB:
1919 if (command == NAND_CMD_READOOB) {
1920 column += mtd->writesize;
1921 command = NAND_CMD_READ0; /* only READ0 is valid */
1922 cmd1 = command << NFCADDR_CMD_CMD1_BIT_POS;
1923 }
1924 if (host->nfc->use_nfc_sram) {
1925 /* Enable Data transfer to sram */
1926 dataen = NFCADDR_CMD_DATAEN;
1927
1928 /* Need enable PMECC now, since NFC will transfer
1929 * data in bus after sending nfc read command.
1930 */
1931 if (chip->ecc.mode == NAND_ECC_HW && host->has_pmecc)
1932 pmecc_enable(host, NAND_ECC_READ);
1933 }
1934
1935 cmd2 = NAND_CMD_READSTART << NFCADDR_CMD_CMD2_BIT_POS;
1936 vcmd2 = NFCADDR_CMD_VCMD2;
1937 break;
1938 /* For prgramming command, the cmd need set to write enable */
1939 case NAND_CMD_PAGEPROG:
1940 case NAND_CMD_SEQIN:
1941 case NAND_CMD_RNDIN:
1942 nfcwr = NFCADDR_CMD_NFCWR;
1943 if (host->nfc->will_write_sram && command == NAND_CMD_SEQIN)
1944 dataen = NFCADDR_CMD_DATAEN;
1945 break;
1946 default:
1947 break;
1948 }
1949
1950 if (do_addr)
1951 acycle = nfc_make_addr(mtd, command, column, page_addr,
1952 &addr1234, &cycle0);
1953
1954 nfc_addr_cmd = cmd1 | cmd2 | vcmd2 | acycle | csid | dataen | nfcwr;
1955 nfc_send_command(host, nfc_addr_cmd, addr1234, cycle0);
1956
1957 /*
1958 * Program and erase have their own busy handlers status, sequential
1959 * in, and deplete1 need no delay.
1960 */
1961 switch (command) {
1962 case NAND_CMD_CACHEDPROG:
1963 case NAND_CMD_PAGEPROG:
1964 case NAND_CMD_ERASE1:
1965 case NAND_CMD_ERASE2:
1966 case NAND_CMD_RNDIN:
1967 case NAND_CMD_STATUS:
1968 case NAND_CMD_RNDOUT:
1969 case NAND_CMD_SEQIN:
1970 case NAND_CMD_READID:
1971 return;
1972
1973 case NAND_CMD_READ0:
1974 if (dataen == NFCADDR_CMD_DATAEN) {
1975 host->nfc->data_in_sram = host->nfc->sram_bank0 +
1976 nfc_get_sram_off(host);
1977 return;
1978 }
1979 /* fall through */
1980 default:
1981 nfc_prepare_interrupt(host, host->nfc->caps->rb_mask);
1982 nfc_wait_interrupt(host, host->nfc->caps->rb_mask);
1983 }
1984}
1985
1986static int nfc_sram_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1987 uint32_t offset, int data_len, const uint8_t *buf,
1988 int oob_required, int page, int cached, int raw)
1989{
1990 int cfg, len;
1991 int status = 0;
1992 struct atmel_nand_host *host = nand_get_controller_data(chip);
1993 void *sram = host->nfc->sram_bank0 + nfc_get_sram_off(host);
1994
1995 /* Subpage write is not supported */
1996 if (offset || (data_len < mtd->writesize))
1997 return -EINVAL;
1998
1999 len = mtd->writesize;
2000 /* Copy page data to sram that will write to nand via NFC */
2001 if (use_dma) {
2002 if (atmel_nand_dma_op(mtd, (void *)buf, len, 0) != 0)
2003 /* Fall back to use cpu copy */
2004 memcpy(sram, buf, len);
2005 } else {
2006 memcpy(sram, buf, len);
2007 }
2008
2009 cfg = nfc_readl(host->nfc->hsmc_regs, CFG);
2010 if (unlikely(raw) && oob_required) {
2011 memcpy(sram + len, chip->oob_poi, mtd->oobsize);
2012 len += mtd->oobsize;
2013 nfc_writel(host->nfc->hsmc_regs, CFG, cfg | NFC_CFG_WSPARE);
2014 } else {
2015 nfc_writel(host->nfc->hsmc_regs, CFG, cfg & ~NFC_CFG_WSPARE);
2016 }
2017
2018 if (chip->ecc.mode == NAND_ECC_HW && host->has_pmecc)
2019 /*
2020 * When use NFC sram, need set up PMECC before send
2021 * NAND_CMD_SEQIN command. Since when the nand command
2022 * is sent, nfc will do transfer from sram and nand.
2023 */
2024 pmecc_enable(host, NAND_ECC_WRITE);
2025
2026 host->nfc->will_write_sram = true;
2027 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
2028 host->nfc->will_write_sram = false;
2029
2030 if (likely(!raw))
2031 /* Need to write ecc into oob */
2032 status = chip->ecc.write_page(mtd, chip, buf, oob_required,
2033 page);
2034
2035 if (status < 0)
2036 return status;
2037
2038 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
2039 status = chip->waitfunc(mtd, chip);
2040
2041 if ((status & NAND_STATUS_FAIL) && (chip->errstat))
2042 status = chip->errstat(mtd, chip, FL_WRITING, status, page);
2043
2044 if (status & NAND_STATUS_FAIL)
2045 return -EIO;
2046
2047 return 0;
2048}
2049
2050static int nfc_sram_init(struct mtd_info *mtd)
2051{
2052 struct nand_chip *chip = mtd_to_nand(mtd);
2053 struct atmel_nand_host *host = nand_get_controller_data(chip);
2054 int res = 0;
2055
2056 /* Initialize the NFC CFG register */
2057 unsigned int cfg_nfc = 0;
2058
2059 /* set page size and oob layout */
2060 switch (mtd->writesize) {
2061 case 512:
2062 cfg_nfc = NFC_CFG_PAGESIZE_512;
2063 break;
2064 case 1024:
2065 cfg_nfc = NFC_CFG_PAGESIZE_1024;
2066 break;
2067 case 2048:
2068 cfg_nfc = NFC_CFG_PAGESIZE_2048;
2069 break;
2070 case 4096:
2071 cfg_nfc = NFC_CFG_PAGESIZE_4096;
2072 break;
2073 case 8192:
2074 cfg_nfc = NFC_CFG_PAGESIZE_8192;
2075 break;
2076 default:
2077 dev_err(host->dev, "Unsupported page size for NFC.\n");
2078 res = -ENXIO;
2079 return res;
2080 }
2081
2082 /* oob bytes size = (NFCSPARESIZE + 1) * 4
2083 * Max support spare size is 512 bytes. */
2084 cfg_nfc |= (((mtd->oobsize / 4) - 1) << NFC_CFG_NFC_SPARESIZE_BIT_POS
2085 & NFC_CFG_NFC_SPARESIZE);
2086 /* default set a max timeout */
2087 cfg_nfc |= NFC_CFG_RSPARE |
2088 NFC_CFG_NFC_DTOCYC | NFC_CFG_NFC_DTOMUL;
2089
2090 nfc_writel(host->nfc->hsmc_regs, CFG, cfg_nfc);
2091
2092 host->nfc->will_write_sram = false;
2093 nfc_set_sram_bank(host, 0);
2094
2095 /* Use Write page with NFC SRAM only for PMECC or ECC NONE. */
2096 if (host->nfc->write_by_sram) {
2097 if ((chip->ecc.mode == NAND_ECC_HW && host->has_pmecc) ||
2098 chip->ecc.mode == NAND_ECC_NONE)
2099 chip->write_page = nfc_sram_write_page;
2100 else
2101 host->nfc->write_by_sram = false;
2102 }
2103
2104 dev_info(host->dev, "Using NFC Sram read %s\n",
2105 host->nfc->write_by_sram ? "and write" : "");
2106 return 0;
2107}
2108
2109static struct platform_driver atmel_nand_nfc_driver;
2110/*
2111 * Probe for the NAND device.
2112 */
2113static int atmel_nand_probe(struct platform_device *pdev)
2114{
2115 struct atmel_nand_host *host;
2116 struct mtd_info *mtd;
2117 struct nand_chip *nand_chip;
2118 struct resource *mem;
2119 int res, irq;
2120
2121 /* Allocate memory for the device structure (and zero it) */
2122 host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL);
2123 if (!host)
2124 return -ENOMEM;
2125
2126 res = platform_driver_register(&atmel_nand_nfc_driver);
2127 if (res)
2128 dev_err(&pdev->dev, "atmel_nand: can't register NFC driver\n");
2129
2130 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2131 host->io_base = devm_ioremap_resource(&pdev->dev, mem);
2132 if (IS_ERR(host->io_base)) {
2133 res = PTR_ERR(host->io_base);
2134 goto err_nand_ioremap;
2135 }
2136 host->io_phys = (dma_addr_t)mem->start;
2137
2138 nand_chip = &host->nand_chip;
2139 mtd = nand_to_mtd(nand_chip);
2140 host->dev = &pdev->dev;
2141 if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) {
2142 nand_set_flash_node(nand_chip, pdev->dev.of_node);
2143 /* Only when CONFIG_OF is enabled of_node can be parsed */
2144 res = atmel_of_init_port(host, pdev->dev.of_node);
2145 if (res)
2146 goto err_nand_ioremap;
2147 } else {
2148 memcpy(&host->board, dev_get_platdata(&pdev->dev),
2149 sizeof(struct atmel_nand_data));
2150 }
2151
2152 /* link the private data structures */
2153 nand_set_controller_data(nand_chip, host);
2154 mtd->dev.parent = &pdev->dev;
2155
2156 /* Set address of NAND IO lines */
2157 nand_chip->IO_ADDR_R = host->io_base;
2158 nand_chip->IO_ADDR_W = host->io_base;
2159
2160 if (nand_nfc.is_initialized) {
2161 /* NFC driver is probed and initialized */
2162 host->nfc = &nand_nfc;
2163
2164 nand_chip->select_chip = nfc_select_chip;
2165 nand_chip->dev_ready = nfc_device_ready;
2166 nand_chip->cmdfunc = nfc_nand_command;
2167
2168 /* Initialize the interrupt for NFC */
2169 irq = platform_get_irq(pdev, 0);
2170 if (irq < 0) {
2171 dev_err(host->dev, "Cannot get HSMC irq!\n");
2172 res = irq;
2173 goto err_nand_ioremap;
2174 }
2175
2176 res = devm_request_irq(&pdev->dev, irq, hsmc_interrupt,
2177 0, "hsmc", host);
2178 if (res) {
2179 dev_err(&pdev->dev, "Unable to request HSMC irq %d\n",
2180 irq);
2181 goto err_nand_ioremap;
2182 }
2183 } else {
2184 res = atmel_nand_set_enable_ready_pins(mtd);
2185 if (res)
2186 goto err_nand_ioremap;
2187
2188 nand_chip->cmd_ctrl = atmel_nand_cmd_ctrl;
2189 }
2190
2191 nand_chip->ecc.mode = host->board.ecc_mode;
2192 nand_chip->chip_delay = 40; /* 40us command delay time */
2193
2194 if (host->board.bus_width_16) /* 16-bit bus width */
2195 nand_chip->options |= NAND_BUSWIDTH_16;
2196
2197 nand_chip->read_buf = atmel_read_buf;
2198 nand_chip->write_buf = atmel_write_buf;
2199
2200 platform_set_drvdata(pdev, host);
2201 atmel_nand_enable(host);
2202
2203 if (gpio_is_valid(host->board.det_pin)) {
2204 res = devm_gpio_request(&pdev->dev,
2205 host->board.det_pin, "nand_det");
2206 if (res < 0) {
2207 dev_err(&pdev->dev,
2208 "can't request det gpio %d\n",
2209 host->board.det_pin);
2210 goto err_no_card;
2211 }
2212
2213 res = gpio_direction_input(host->board.det_pin);
2214 if (res < 0) {
2215 dev_err(&pdev->dev,
2216 "can't request input direction det gpio %d\n",
2217 host->board.det_pin);
2218 goto err_no_card;
2219 }
2220
2221 if (gpio_get_value(host->board.det_pin)) {
2222 dev_info(&pdev->dev, "No SmartMedia card inserted.\n");
2223 res = -ENXIO;
2224 goto err_no_card;
2225 }
2226 }
2227
2228 if (host->board.on_flash_bbt || on_flash_bbt) {
2229 dev_info(&pdev->dev, "Use On Flash BBT\n");
2230 nand_chip->bbt_options |= NAND_BBT_USE_FLASH;
2231 }
2232
2233 if (!host->board.has_dma)
2234 use_dma = 0;
2235
2236 if (use_dma) {
2237 dma_cap_mask_t mask;
2238
2239 dma_cap_zero(mask);
2240 dma_cap_set(DMA_MEMCPY, mask);
2241 host->dma_chan = dma_request_channel(mask, NULL, NULL);
2242 if (!host->dma_chan) {
2243 dev_err(host->dev, "Failed to request DMA channel\n");
2244 use_dma = 0;
2245 }
2246 }
2247 if (use_dma)
2248 dev_info(host->dev, "Using %s for DMA transfers.\n",
2249 dma_chan_name(host->dma_chan));
2250 else
2251 dev_info(host->dev, "No DMA support for NAND access.\n");
2252
2253 /* first scan to find the device and get the page size */
2254 if (nand_scan_ident(mtd, 1, NULL)) {
2255 res = -ENXIO;
2256 goto err_scan_ident;
2257 }
2258
2259 if (nand_chip->ecc.mode == NAND_ECC_HW) {
2260 if (host->has_pmecc)
2261 res = atmel_pmecc_nand_init_params(pdev, host);
2262 else
2263 res = atmel_hw_nand_init_params(pdev, host);
2264
2265 if (res != 0)
2266 goto err_hw_ecc;
2267 }
2268
2269 /* initialize the nfc configuration register */
2270 if (host->nfc && host->nfc->use_nfc_sram) {
2271 res = nfc_sram_init(mtd);
2272 if (res) {
2273 host->nfc->use_nfc_sram = false;
2274 dev_err(host->dev, "Disable use nfc sram for data transfer.\n");
2275 }
2276 }
2277
2278 /* second phase scan */
2279 if (nand_scan_tail(mtd)) {
2280 res = -ENXIO;
2281 goto err_scan_tail;
2282 }
2283
2284 mtd->name = "atmel_nand";
2285 res = mtd_device_register(mtd, host->board.parts,
2286 host->board.num_parts);
2287 if (!res)
2288 return res;
2289
2290err_scan_tail:
2291 if (host->has_pmecc && host->nand_chip.ecc.mode == NAND_ECC_HW)
2292 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
2293err_hw_ecc:
2294err_scan_ident:
2295err_no_card:
2296 atmel_nand_disable(host);
2297 if (host->dma_chan)
2298 dma_release_channel(host->dma_chan);
2299err_nand_ioremap:
2300 return res;
2301}
2302
2303/*
2304 * Remove a NAND device.
2305 */
2306static int atmel_nand_remove(struct platform_device *pdev)
2307{
2308 struct atmel_nand_host *host = platform_get_drvdata(pdev);
2309 struct mtd_info *mtd = nand_to_mtd(&host->nand_chip);
2310
2311 nand_release(mtd);
2312
2313 atmel_nand_disable(host);
2314
2315 if (host->has_pmecc && host->nand_chip.ecc.mode == NAND_ECC_HW) {
2316 pmecc_writel(host->ecc, CTRL, PMECC_CTRL_DISABLE);
2317 pmerrloc_writel(host->pmerrloc_base, ELDIS,
2318 PMERRLOC_DISABLE);
2319 }
2320
2321 if (host->dma_chan)
2322 dma_release_channel(host->dma_chan);
2323
2324 platform_driver_unregister(&atmel_nand_nfc_driver);
2325
2326 return 0;
2327}
2328
2329/*
2330 * AT91RM9200 does not have PMECC or PMECC Errloc peripherals for
2331 * BCH ECC. Combined with the "atmel,has-pmecc", it is used to describe
2332 * devices from the SAM9 family that have those.
2333 */
2334static const struct atmel_nand_caps at91rm9200_caps = {
2335 .pmecc_correct_erase_page = false,
2336 .pmecc_max_correction = 24,
2337};
2338
2339static const struct atmel_nand_caps sama5d4_caps = {
2340 .pmecc_correct_erase_page = true,
2341 .pmecc_max_correction = 24,
2342};
2343
2344/*
2345 * The PMECC Errloc controller starting in SAMA5D2 is not compatible,
2346 * as the increased correction strength requires more registers.
2347 */
2348static const struct atmel_nand_caps sama5d2_caps = {
2349 .pmecc_correct_erase_page = true,
2350 .pmecc_max_correction = 32,
2351};
2352
2353static const struct of_device_id atmel_nand_dt_ids[] = {
2354 { .compatible = "atmel,at91rm9200-nand", .data = &at91rm9200_caps },
2355 { .compatible = "atmel,sama5d4-nand", .data = &sama5d4_caps },
2356 { .compatible = "atmel,sama5d2-nand", .data = &sama5d2_caps },
2357 { /* sentinel */ }
2358};
2359
2360MODULE_DEVICE_TABLE(of, atmel_nand_dt_ids);
2361
2362static int atmel_nand_nfc_probe(struct platform_device *pdev)
2363{
2364 struct atmel_nfc *nfc = &nand_nfc;
2365 struct resource *nfc_cmd_regs, *nfc_hsmc_regs, *nfc_sram;
2366 int ret;
2367
2368 nfc_cmd_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2369 nfc->base_cmd_regs = devm_ioremap_resource(&pdev->dev, nfc_cmd_regs);
2370 if (IS_ERR(nfc->base_cmd_regs))
2371 return PTR_ERR(nfc->base_cmd_regs);
2372
2373 nfc_hsmc_regs = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2374 nfc->hsmc_regs = devm_ioremap_resource(&pdev->dev, nfc_hsmc_regs);
2375 if (IS_ERR(nfc->hsmc_regs))
2376 return PTR_ERR(nfc->hsmc_regs);
2377
2378 nfc_sram = platform_get_resource(pdev, IORESOURCE_MEM, 2);
2379 if (nfc_sram) {
2380 nfc->sram_bank0 = (void * __force)
2381 devm_ioremap_resource(&pdev->dev, nfc_sram);
2382 if (IS_ERR(nfc->sram_bank0)) {
2383 dev_warn(&pdev->dev, "Fail to ioremap the NFC sram with error: %ld. So disable NFC sram.\n",
2384 PTR_ERR(nfc->sram_bank0));
2385 } else {
2386 nfc->use_nfc_sram = true;
2387 nfc->sram_bank0_phys = (dma_addr_t)nfc_sram->start;
2388
2389 if (pdev->dev.of_node)
2390 nfc->write_by_sram = of_property_read_bool(
2391 pdev->dev.of_node,
2392 "atmel,write-by-sram");
2393 }
2394 }
2395
2396 nfc->caps = (const struct atmel_nand_nfc_caps *)
2397 of_device_get_match_data(&pdev->dev);
2398 if (!nfc->caps)
2399 return -ENODEV;
2400
2401 nfc_writel(nfc->hsmc_regs, IDR, 0xffffffff);
2402 nfc_readl(nfc->hsmc_regs, SR); /* clear the NFC_SR */
2403
2404 nfc->clk = devm_clk_get(&pdev->dev, NULL);
2405 if (!IS_ERR(nfc->clk)) {
2406 ret = clk_prepare_enable(nfc->clk);
2407 if (ret)
2408 return ret;
2409 } else {
2410 dev_warn(&pdev->dev, "NFC clock missing, update your Device Tree");
2411 }
2412
2413 nfc->is_initialized = true;
2414 dev_info(&pdev->dev, "NFC is probed.\n");
2415
2416 return 0;
2417}
2418
2419static int atmel_nand_nfc_remove(struct platform_device *pdev)
2420{
2421 struct atmel_nfc *nfc = &nand_nfc;
2422
2423 if (!IS_ERR(nfc->clk))
2424 clk_disable_unprepare(nfc->clk);
2425
2426 return 0;
2427}
2428
2429static const struct atmel_nand_nfc_caps sama5d3_nfc_caps = {
2430 .rb_mask = NFC_SR_RB_EDGE0,
2431};
2432
2433static const struct atmel_nand_nfc_caps sama5d4_nfc_caps = {
2434 .rb_mask = NFC_SR_RB_EDGE3,
2435};
2436
2437static const struct of_device_id atmel_nand_nfc_match[] = {
2438 { .compatible = "atmel,sama5d3-nfc", .data = &sama5d3_nfc_caps },
2439 { .compatible = "atmel,sama5d4-nfc", .data = &sama5d4_nfc_caps },
2440 { /* sentinel */ }
2441};
2442MODULE_DEVICE_TABLE(of, atmel_nand_nfc_match);
2443
2444static struct platform_driver atmel_nand_nfc_driver = {
2445 .driver = {
2446 .name = "atmel_nand_nfc",
2447 .of_match_table = of_match_ptr(atmel_nand_nfc_match),
2448 },
2449 .probe = atmel_nand_nfc_probe,
2450 .remove = atmel_nand_nfc_remove,
2451};
2452
2453static struct platform_driver atmel_nand_driver = {
2454 .probe = atmel_nand_probe,
2455 .remove = atmel_nand_remove,
2456 .driver = {
2457 .name = "atmel_nand",
2458 .of_match_table = of_match_ptr(atmel_nand_dt_ids),
2459 },
2460};
2461
2462module_platform_driver(atmel_nand_driver);
2463
2464MODULE_LICENSE("GPL");
2465MODULE_AUTHOR("Rick Bronson");
2466MODULE_DESCRIPTION("NAND/SmartMedia driver for AT91 / AVR32");
2467MODULE_ALIAS("platform:atmel_nand");