Loading...
Note: File does not exist in v6.8.
1/*
2 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com>
3 * Copyright © 2004 Micron Technology Inc.
4 * Copyright © 2004 David Brownell
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/platform_device.h>
12#include <linux/dmaengine.h>
13#include <linux/dma-mapping.h>
14#include <linux/delay.h>
15#include <linux/module.h>
16#include <linux/interrupt.h>
17#include <linux/jiffies.h>
18#include <linux/sched.h>
19#include <linux/mtd/mtd.h>
20#include <linux/mtd/nand.h>
21#include <linux/mtd/partitions.h>
22#include <linux/omap-dma.h>
23#include <linux/io.h>
24#include <linux/slab.h>
25#include <linux/of.h>
26#include <linux/of_device.h>
27
28#include <linux/mtd/nand_bch.h>
29#include <linux/platform_data/elm.h>
30
31#include <linux/platform_data/mtd-nand-omap2.h>
32
33#define DRIVER_NAME "omap2-nand"
34#define OMAP_NAND_TIMEOUT_MS 5000
35
36#define NAND_Ecc_P1e (1 << 0)
37#define NAND_Ecc_P2e (1 << 1)
38#define NAND_Ecc_P4e (1 << 2)
39#define NAND_Ecc_P8e (1 << 3)
40#define NAND_Ecc_P16e (1 << 4)
41#define NAND_Ecc_P32e (1 << 5)
42#define NAND_Ecc_P64e (1 << 6)
43#define NAND_Ecc_P128e (1 << 7)
44#define NAND_Ecc_P256e (1 << 8)
45#define NAND_Ecc_P512e (1 << 9)
46#define NAND_Ecc_P1024e (1 << 10)
47#define NAND_Ecc_P2048e (1 << 11)
48
49#define NAND_Ecc_P1o (1 << 16)
50#define NAND_Ecc_P2o (1 << 17)
51#define NAND_Ecc_P4o (1 << 18)
52#define NAND_Ecc_P8o (1 << 19)
53#define NAND_Ecc_P16o (1 << 20)
54#define NAND_Ecc_P32o (1 << 21)
55#define NAND_Ecc_P64o (1 << 22)
56#define NAND_Ecc_P128o (1 << 23)
57#define NAND_Ecc_P256o (1 << 24)
58#define NAND_Ecc_P512o (1 << 25)
59#define NAND_Ecc_P1024o (1 << 26)
60#define NAND_Ecc_P2048o (1 << 27)
61
62#define TF(value) (value ? 1 : 0)
63
64#define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0)
65#define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1)
66#define P1e(a) (TF(a & NAND_Ecc_P1e) << 2)
67#define P1o(a) (TF(a & NAND_Ecc_P1o) << 3)
68#define P2e(a) (TF(a & NAND_Ecc_P2e) << 4)
69#define P2o(a) (TF(a & NAND_Ecc_P2o) << 5)
70#define P4e(a) (TF(a & NAND_Ecc_P4e) << 6)
71#define P4o(a) (TF(a & NAND_Ecc_P4o) << 7)
72
73#define P8e(a) (TF(a & NAND_Ecc_P8e) << 0)
74#define P8o(a) (TF(a & NAND_Ecc_P8o) << 1)
75#define P16e(a) (TF(a & NAND_Ecc_P16e) << 2)
76#define P16o(a) (TF(a & NAND_Ecc_P16o) << 3)
77#define P32e(a) (TF(a & NAND_Ecc_P32e) << 4)
78#define P32o(a) (TF(a & NAND_Ecc_P32o) << 5)
79#define P64e(a) (TF(a & NAND_Ecc_P64e) << 6)
80#define P64o(a) (TF(a & NAND_Ecc_P64o) << 7)
81
82#define P128e(a) (TF(a & NAND_Ecc_P128e) << 0)
83#define P128o(a) (TF(a & NAND_Ecc_P128o) << 1)
84#define P256e(a) (TF(a & NAND_Ecc_P256e) << 2)
85#define P256o(a) (TF(a & NAND_Ecc_P256o) << 3)
86#define P512e(a) (TF(a & NAND_Ecc_P512e) << 4)
87#define P512o(a) (TF(a & NAND_Ecc_P512o) << 5)
88#define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6)
89#define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7)
90
91#define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0)
92#define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1)
93#define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2)
94#define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3)
95#define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4)
96#define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5)
97#define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6)
98#define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7)
99
100#define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0)
101#define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1)
102
103#define PREFETCH_CONFIG1_CS_SHIFT 24
104#define ECC_CONFIG_CS_SHIFT 1
105#define CS_MASK 0x7
106#define ENABLE_PREFETCH (0x1 << 7)
107#define DMA_MPU_MODE_SHIFT 2
108#define ECCSIZE0_SHIFT 12
109#define ECCSIZE1_SHIFT 22
110#define ECC1RESULTSIZE 0x1
111#define ECCCLEAR 0x100
112#define ECC1 0x1
113#define PREFETCH_FIFOTHRESHOLD_MAX 0x40
114#define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8)
115#define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff)
116#define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F)
117#define STATUS_BUFF_EMPTY 0x00000001
118
119#define OMAP24XX_DMA_GPMC 4
120
121#define SECTOR_BYTES 512
122/* 4 bit padding to make byte aligned, 56 = 52 + 4 */
123#define BCH4_BIT_PAD 4
124
125/* GPMC ecc engine settings for read */
126#define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */
127#define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */
128#define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */
129#define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */
130#define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */
131
132/* GPMC ecc engine settings for write */
133#define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */
134#define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */
135#define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */
136
137#define BADBLOCK_MARKER_LENGTH 2
138
139static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55,
140 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78,
141 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93,
142 0x07, 0x0e};
143static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc,
144 0xac, 0x6b, 0xff, 0x99, 0x7b};
145static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10};
146
147/* Shared among all NAND instances to synchronize access to the ECC Engine */
148static struct nand_hw_control omap_gpmc_controller = {
149 .lock = __SPIN_LOCK_UNLOCKED(omap_gpmc_controller.lock),
150 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(omap_gpmc_controller.wq),
151};
152
153struct omap_nand_info {
154 struct omap_nand_platform_data *pdata;
155 struct nand_chip nand;
156 struct platform_device *pdev;
157
158 int gpmc_cs;
159 unsigned long phys_base;
160 enum omap_ecc ecc_opt;
161 struct completion comp;
162 struct dma_chan *dma;
163 int gpmc_irq_fifo;
164 int gpmc_irq_count;
165 enum {
166 OMAP_NAND_IO_READ = 0, /* read */
167 OMAP_NAND_IO_WRITE, /* write */
168 } iomode;
169 u_char *buf;
170 int buf_len;
171 struct gpmc_nand_regs reg;
172 /* generated at runtime depending on ECC algorithm and layout selected */
173 struct nand_ecclayout oobinfo;
174 /* fields specific for BCHx_HW ECC scheme */
175 struct device *elm_dev;
176 struct device_node *of_node;
177};
178
179static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd)
180{
181 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand);
182}
183
184/**
185 * omap_prefetch_enable - configures and starts prefetch transfer
186 * @cs: cs (chip select) number
187 * @fifo_th: fifo threshold to be used for read/ write
188 * @dma_mode: dma mode enable (1) or disable (0)
189 * @u32_count: number of bytes to be transferred
190 * @is_write: prefetch read(0) or write post(1) mode
191 */
192static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode,
193 unsigned int u32_count, int is_write, struct omap_nand_info *info)
194{
195 u32 val;
196
197 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX)
198 return -1;
199
200 if (readl(info->reg.gpmc_prefetch_control))
201 return -EBUSY;
202
203 /* Set the amount of bytes to be prefetched */
204 writel(u32_count, info->reg.gpmc_prefetch_config2);
205
206 /* Set dma/mpu mode, the prefetch read / post write and
207 * enable the engine. Set which cs is has requested for.
208 */
209 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) |
210 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH |
211 (dma_mode << DMA_MPU_MODE_SHIFT) | (0x1 & is_write));
212 writel(val, info->reg.gpmc_prefetch_config1);
213
214 /* Start the prefetch engine */
215 writel(0x1, info->reg.gpmc_prefetch_control);
216
217 return 0;
218}
219
220/**
221 * omap_prefetch_reset - disables and stops the prefetch engine
222 */
223static int omap_prefetch_reset(int cs, struct omap_nand_info *info)
224{
225 u32 config1;
226
227 /* check if the same module/cs is trying to reset */
228 config1 = readl(info->reg.gpmc_prefetch_config1);
229 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs)
230 return -EINVAL;
231
232 /* Stop the PFPW engine */
233 writel(0x0, info->reg.gpmc_prefetch_control);
234
235 /* Reset/disable the PFPW engine */
236 writel(0x0, info->reg.gpmc_prefetch_config1);
237
238 return 0;
239}
240
241/**
242 * omap_hwcontrol - hardware specific access to control-lines
243 * @mtd: MTD device structure
244 * @cmd: command to device
245 * @ctrl:
246 * NAND_NCE: bit 0 -> don't care
247 * NAND_CLE: bit 1 -> Command Latch
248 * NAND_ALE: bit 2 -> Address Latch
249 *
250 * NOTE: boards may use different bits for these!!
251 */
252static void omap_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl)
253{
254 struct omap_nand_info *info = mtd_to_omap(mtd);
255
256 if (cmd != NAND_CMD_NONE) {
257 if (ctrl & NAND_CLE)
258 writeb(cmd, info->reg.gpmc_nand_command);
259
260 else if (ctrl & NAND_ALE)
261 writeb(cmd, info->reg.gpmc_nand_address);
262
263 else /* NAND_NCE */
264 writeb(cmd, info->reg.gpmc_nand_data);
265 }
266}
267
268/**
269 * omap_read_buf8 - read data from NAND controller into buffer
270 * @mtd: MTD device structure
271 * @buf: buffer to store date
272 * @len: number of bytes to read
273 */
274static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len)
275{
276 struct nand_chip *nand = mtd_to_nand(mtd);
277
278 ioread8_rep(nand->IO_ADDR_R, buf, len);
279}
280
281/**
282 * omap_write_buf8 - write buffer to NAND controller
283 * @mtd: MTD device structure
284 * @buf: data buffer
285 * @len: number of bytes to write
286 */
287static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len)
288{
289 struct omap_nand_info *info = mtd_to_omap(mtd);
290 u_char *p = (u_char *)buf;
291 u32 status = 0;
292
293 while (len--) {
294 iowrite8(*p++, info->nand.IO_ADDR_W);
295 /* wait until buffer is available for write */
296 do {
297 status = readl(info->reg.gpmc_status) &
298 STATUS_BUFF_EMPTY;
299 } while (!status);
300 }
301}
302
303/**
304 * omap_read_buf16 - read data from NAND controller into buffer
305 * @mtd: MTD device structure
306 * @buf: buffer to store date
307 * @len: number of bytes to read
308 */
309static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len)
310{
311 struct nand_chip *nand = mtd_to_nand(mtd);
312
313 ioread16_rep(nand->IO_ADDR_R, buf, len / 2);
314}
315
316/**
317 * omap_write_buf16 - write buffer to NAND controller
318 * @mtd: MTD device structure
319 * @buf: data buffer
320 * @len: number of bytes to write
321 */
322static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len)
323{
324 struct omap_nand_info *info = mtd_to_omap(mtd);
325 u16 *p = (u16 *) buf;
326 u32 status = 0;
327 /* FIXME try bursts of writesw() or DMA ... */
328 len >>= 1;
329
330 while (len--) {
331 iowrite16(*p++, info->nand.IO_ADDR_W);
332 /* wait until buffer is available for write */
333 do {
334 status = readl(info->reg.gpmc_status) &
335 STATUS_BUFF_EMPTY;
336 } while (!status);
337 }
338}
339
340/**
341 * omap_read_buf_pref - read data from NAND controller into buffer
342 * @mtd: MTD device structure
343 * @buf: buffer to store date
344 * @len: number of bytes to read
345 */
346static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len)
347{
348 struct omap_nand_info *info = mtd_to_omap(mtd);
349 uint32_t r_count = 0;
350 int ret = 0;
351 u32 *p = (u32 *)buf;
352
353 /* take care of subpage reads */
354 if (len % 4) {
355 if (info->nand.options & NAND_BUSWIDTH_16)
356 omap_read_buf16(mtd, buf, len % 4);
357 else
358 omap_read_buf8(mtd, buf, len % 4);
359 p = (u32 *) (buf + len % 4);
360 len -= len % 4;
361 }
362
363 /* configure and start prefetch transfer */
364 ret = omap_prefetch_enable(info->gpmc_cs,
365 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info);
366 if (ret) {
367 /* PFPW engine is busy, use cpu copy method */
368 if (info->nand.options & NAND_BUSWIDTH_16)
369 omap_read_buf16(mtd, (u_char *)p, len);
370 else
371 omap_read_buf8(mtd, (u_char *)p, len);
372 } else {
373 do {
374 r_count = readl(info->reg.gpmc_prefetch_status);
375 r_count = PREFETCH_STATUS_FIFO_CNT(r_count);
376 r_count = r_count >> 2;
377 ioread32_rep(info->nand.IO_ADDR_R, p, r_count);
378 p += r_count;
379 len -= r_count << 2;
380 } while (len);
381 /* disable and stop the PFPW engine */
382 omap_prefetch_reset(info->gpmc_cs, info);
383 }
384}
385
386/**
387 * omap_write_buf_pref - write buffer to NAND controller
388 * @mtd: MTD device structure
389 * @buf: data buffer
390 * @len: number of bytes to write
391 */
392static void omap_write_buf_pref(struct mtd_info *mtd,
393 const u_char *buf, int len)
394{
395 struct omap_nand_info *info = mtd_to_omap(mtd);
396 uint32_t w_count = 0;
397 int i = 0, ret = 0;
398 u16 *p = (u16 *)buf;
399 unsigned long tim, limit;
400 u32 val;
401
402 /* take care of subpage writes */
403 if (len % 2 != 0) {
404 writeb(*buf, info->nand.IO_ADDR_W);
405 p = (u16 *)(buf + 1);
406 len--;
407 }
408
409 /* configure and start prefetch transfer */
410 ret = omap_prefetch_enable(info->gpmc_cs,
411 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info);
412 if (ret) {
413 /* PFPW engine is busy, use cpu copy method */
414 if (info->nand.options & NAND_BUSWIDTH_16)
415 omap_write_buf16(mtd, (u_char *)p, len);
416 else
417 omap_write_buf8(mtd, (u_char *)p, len);
418 } else {
419 while (len) {
420 w_count = readl(info->reg.gpmc_prefetch_status);
421 w_count = PREFETCH_STATUS_FIFO_CNT(w_count);
422 w_count = w_count >> 1;
423 for (i = 0; (i < w_count) && len; i++, len -= 2)
424 iowrite16(*p++, info->nand.IO_ADDR_W);
425 }
426 /* wait for data to flushed-out before reset the prefetch */
427 tim = 0;
428 limit = (loops_per_jiffy *
429 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
430 do {
431 cpu_relax();
432 val = readl(info->reg.gpmc_prefetch_status);
433 val = PREFETCH_STATUS_COUNT(val);
434 } while (val && (tim++ < limit));
435
436 /* disable and stop the PFPW engine */
437 omap_prefetch_reset(info->gpmc_cs, info);
438 }
439}
440
441/*
442 * omap_nand_dma_callback: callback on the completion of dma transfer
443 * @data: pointer to completion data structure
444 */
445static void omap_nand_dma_callback(void *data)
446{
447 complete((struct completion *) data);
448}
449
450/*
451 * omap_nand_dma_transfer: configure and start dma transfer
452 * @mtd: MTD device structure
453 * @addr: virtual address in RAM of source/destination
454 * @len: number of data bytes to be transferred
455 * @is_write: flag for read/write operation
456 */
457static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr,
458 unsigned int len, int is_write)
459{
460 struct omap_nand_info *info = mtd_to_omap(mtd);
461 struct dma_async_tx_descriptor *tx;
462 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE :
463 DMA_FROM_DEVICE;
464 struct scatterlist sg;
465 unsigned long tim, limit;
466 unsigned n;
467 int ret;
468 u32 val;
469
470 if (addr >= high_memory) {
471 struct page *p1;
472
473 if (((size_t)addr & PAGE_MASK) !=
474 ((size_t)(addr + len - 1) & PAGE_MASK))
475 goto out_copy;
476 p1 = vmalloc_to_page(addr);
477 if (!p1)
478 goto out_copy;
479 addr = page_address(p1) + ((size_t)addr & ~PAGE_MASK);
480 }
481
482 sg_init_one(&sg, addr, len);
483 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir);
484 if (n == 0) {
485 dev_err(&info->pdev->dev,
486 "Couldn't DMA map a %d byte buffer\n", len);
487 goto out_copy;
488 }
489
490 tx = dmaengine_prep_slave_sg(info->dma, &sg, n,
491 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
492 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
493 if (!tx)
494 goto out_copy_unmap;
495
496 tx->callback = omap_nand_dma_callback;
497 tx->callback_param = &info->comp;
498 dmaengine_submit(tx);
499
500 /* configure and start prefetch transfer */
501 ret = omap_prefetch_enable(info->gpmc_cs,
502 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info);
503 if (ret)
504 /* PFPW engine is busy, use cpu copy method */
505 goto out_copy_unmap;
506
507 init_completion(&info->comp);
508 dma_async_issue_pending(info->dma);
509
510 /* setup and start DMA using dma_addr */
511 wait_for_completion(&info->comp);
512 tim = 0;
513 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
514
515 do {
516 cpu_relax();
517 val = readl(info->reg.gpmc_prefetch_status);
518 val = PREFETCH_STATUS_COUNT(val);
519 } while (val && (tim++ < limit));
520
521 /* disable and stop the PFPW engine */
522 omap_prefetch_reset(info->gpmc_cs, info);
523
524 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
525 return 0;
526
527out_copy_unmap:
528 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
529out_copy:
530 if (info->nand.options & NAND_BUSWIDTH_16)
531 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len)
532 : omap_write_buf16(mtd, (u_char *) addr, len);
533 else
534 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len)
535 : omap_write_buf8(mtd, (u_char *) addr, len);
536 return 0;
537}
538
539/**
540 * omap_read_buf_dma_pref - read data from NAND controller into buffer
541 * @mtd: MTD device structure
542 * @buf: buffer to store date
543 * @len: number of bytes to read
544 */
545static void omap_read_buf_dma_pref(struct mtd_info *mtd, u_char *buf, int len)
546{
547 if (len <= mtd->oobsize)
548 omap_read_buf_pref(mtd, buf, len);
549 else
550 /* start transfer in DMA mode */
551 omap_nand_dma_transfer(mtd, buf, len, 0x0);
552}
553
554/**
555 * omap_write_buf_dma_pref - write buffer to NAND controller
556 * @mtd: MTD device structure
557 * @buf: data buffer
558 * @len: number of bytes to write
559 */
560static void omap_write_buf_dma_pref(struct mtd_info *mtd,
561 const u_char *buf, int len)
562{
563 if (len <= mtd->oobsize)
564 omap_write_buf_pref(mtd, buf, len);
565 else
566 /* start transfer in DMA mode */
567 omap_nand_dma_transfer(mtd, (u_char *) buf, len, 0x1);
568}
569
570/*
571 * omap_nand_irq - GPMC irq handler
572 * @this_irq: gpmc irq number
573 * @dev: omap_nand_info structure pointer is passed here
574 */
575static irqreturn_t omap_nand_irq(int this_irq, void *dev)
576{
577 struct omap_nand_info *info = (struct omap_nand_info *) dev;
578 u32 bytes;
579
580 bytes = readl(info->reg.gpmc_prefetch_status);
581 bytes = PREFETCH_STATUS_FIFO_CNT(bytes);
582 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */
583 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */
584 if (this_irq == info->gpmc_irq_count)
585 goto done;
586
587 if (info->buf_len && (info->buf_len < bytes))
588 bytes = info->buf_len;
589 else if (!info->buf_len)
590 bytes = 0;
591 iowrite32_rep(info->nand.IO_ADDR_W,
592 (u32 *)info->buf, bytes >> 2);
593 info->buf = info->buf + bytes;
594 info->buf_len -= bytes;
595
596 } else {
597 ioread32_rep(info->nand.IO_ADDR_R,
598 (u32 *)info->buf, bytes >> 2);
599 info->buf = info->buf + bytes;
600
601 if (this_irq == info->gpmc_irq_count)
602 goto done;
603 }
604
605 return IRQ_HANDLED;
606
607done:
608 complete(&info->comp);
609
610 disable_irq_nosync(info->gpmc_irq_fifo);
611 disable_irq_nosync(info->gpmc_irq_count);
612
613 return IRQ_HANDLED;
614}
615
616/*
617 * omap_read_buf_irq_pref - read data from NAND controller into buffer
618 * @mtd: MTD device structure
619 * @buf: buffer to store date
620 * @len: number of bytes to read
621 */
622static void omap_read_buf_irq_pref(struct mtd_info *mtd, u_char *buf, int len)
623{
624 struct omap_nand_info *info = mtd_to_omap(mtd);
625 int ret = 0;
626
627 if (len <= mtd->oobsize) {
628 omap_read_buf_pref(mtd, buf, len);
629 return;
630 }
631
632 info->iomode = OMAP_NAND_IO_READ;
633 info->buf = buf;
634 init_completion(&info->comp);
635
636 /* configure and start prefetch transfer */
637 ret = omap_prefetch_enable(info->gpmc_cs,
638 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info);
639 if (ret)
640 /* PFPW engine is busy, use cpu copy method */
641 goto out_copy;
642
643 info->buf_len = len;
644
645 enable_irq(info->gpmc_irq_count);
646 enable_irq(info->gpmc_irq_fifo);
647
648 /* waiting for read to complete */
649 wait_for_completion(&info->comp);
650
651 /* disable and stop the PFPW engine */
652 omap_prefetch_reset(info->gpmc_cs, info);
653 return;
654
655out_copy:
656 if (info->nand.options & NAND_BUSWIDTH_16)
657 omap_read_buf16(mtd, buf, len);
658 else
659 omap_read_buf8(mtd, buf, len);
660}
661
662/*
663 * omap_write_buf_irq_pref - write buffer to NAND controller
664 * @mtd: MTD device structure
665 * @buf: data buffer
666 * @len: number of bytes to write
667 */
668static void omap_write_buf_irq_pref(struct mtd_info *mtd,
669 const u_char *buf, int len)
670{
671 struct omap_nand_info *info = mtd_to_omap(mtd);
672 int ret = 0;
673 unsigned long tim, limit;
674 u32 val;
675
676 if (len <= mtd->oobsize) {
677 omap_write_buf_pref(mtd, buf, len);
678 return;
679 }
680
681 info->iomode = OMAP_NAND_IO_WRITE;
682 info->buf = (u_char *) buf;
683 init_completion(&info->comp);
684
685 /* configure and start prefetch transfer : size=24 */
686 ret = omap_prefetch_enable(info->gpmc_cs,
687 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info);
688 if (ret)
689 /* PFPW engine is busy, use cpu copy method */
690 goto out_copy;
691
692 info->buf_len = len;
693
694 enable_irq(info->gpmc_irq_count);
695 enable_irq(info->gpmc_irq_fifo);
696
697 /* waiting for write to complete */
698 wait_for_completion(&info->comp);
699
700 /* wait for data to flushed-out before reset the prefetch */
701 tim = 0;
702 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
703 do {
704 val = readl(info->reg.gpmc_prefetch_status);
705 val = PREFETCH_STATUS_COUNT(val);
706 cpu_relax();
707 } while (val && (tim++ < limit));
708
709 /* disable and stop the PFPW engine */
710 omap_prefetch_reset(info->gpmc_cs, info);
711 return;
712
713out_copy:
714 if (info->nand.options & NAND_BUSWIDTH_16)
715 omap_write_buf16(mtd, buf, len);
716 else
717 omap_write_buf8(mtd, buf, len);
718}
719
720/**
721 * gen_true_ecc - This function will generate true ECC value
722 * @ecc_buf: buffer to store ecc code
723 *
724 * This generated true ECC value can be used when correcting
725 * data read from NAND flash memory core
726 */
727static void gen_true_ecc(u8 *ecc_buf)
728{
729 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) |
730 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8);
731
732 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) |
733 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp));
734 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) |
735 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp));
736 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) |
737 P1e(tmp) | P2048o(tmp) | P2048e(tmp));
738}
739
740/**
741 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data
742 * @ecc_data1: ecc code from nand spare area
743 * @ecc_data2: ecc code from hardware register obtained from hardware ecc
744 * @page_data: page data
745 *
746 * This function compares two ECC's and indicates if there is an error.
747 * If the error can be corrected it will be corrected to the buffer.
748 * If there is no error, %0 is returned. If there is an error but it
749 * was corrected, %1 is returned. Otherwise, %-1 is returned.
750 */
751static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */
752 u8 *ecc_data2, /* read from register */
753 u8 *page_data)
754{
755 uint i;
756 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8];
757 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8];
758 u8 ecc_bit[24];
759 u8 ecc_sum = 0;
760 u8 find_bit = 0;
761 uint find_byte = 0;
762 int isEccFF;
763
764 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF);
765
766 gen_true_ecc(ecc_data1);
767 gen_true_ecc(ecc_data2);
768
769 for (i = 0; i <= 2; i++) {
770 *(ecc_data1 + i) = ~(*(ecc_data1 + i));
771 *(ecc_data2 + i) = ~(*(ecc_data2 + i));
772 }
773
774 for (i = 0; i < 8; i++) {
775 tmp0_bit[i] = *ecc_data1 % 2;
776 *ecc_data1 = *ecc_data1 / 2;
777 }
778
779 for (i = 0; i < 8; i++) {
780 tmp1_bit[i] = *(ecc_data1 + 1) % 2;
781 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2;
782 }
783
784 for (i = 0; i < 8; i++) {
785 tmp2_bit[i] = *(ecc_data1 + 2) % 2;
786 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2;
787 }
788
789 for (i = 0; i < 8; i++) {
790 comp0_bit[i] = *ecc_data2 % 2;
791 *ecc_data2 = *ecc_data2 / 2;
792 }
793
794 for (i = 0; i < 8; i++) {
795 comp1_bit[i] = *(ecc_data2 + 1) % 2;
796 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2;
797 }
798
799 for (i = 0; i < 8; i++) {
800 comp2_bit[i] = *(ecc_data2 + 2) % 2;
801 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2;
802 }
803
804 for (i = 0; i < 6; i++)
805 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2];
806
807 for (i = 0; i < 8; i++)
808 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i];
809
810 for (i = 0; i < 8; i++)
811 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i];
812
813 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0];
814 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1];
815
816 for (i = 0; i < 24; i++)
817 ecc_sum += ecc_bit[i];
818
819 switch (ecc_sum) {
820 case 0:
821 /* Not reached because this function is not called if
822 * ECC values are equal
823 */
824 return 0;
825
826 case 1:
827 /* Uncorrectable error */
828 pr_debug("ECC UNCORRECTED_ERROR 1\n");
829 return -EBADMSG;
830
831 case 11:
832 /* UN-Correctable error */
833 pr_debug("ECC UNCORRECTED_ERROR B\n");
834 return -EBADMSG;
835
836 case 12:
837 /* Correctable error */
838 find_byte = (ecc_bit[23] << 8) +
839 (ecc_bit[21] << 7) +
840 (ecc_bit[19] << 6) +
841 (ecc_bit[17] << 5) +
842 (ecc_bit[15] << 4) +
843 (ecc_bit[13] << 3) +
844 (ecc_bit[11] << 2) +
845 (ecc_bit[9] << 1) +
846 ecc_bit[7];
847
848 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1];
849
850 pr_debug("Correcting single bit ECC error at offset: "
851 "%d, bit: %d\n", find_byte, find_bit);
852
853 page_data[find_byte] ^= (1 << find_bit);
854
855 return 1;
856 default:
857 if (isEccFF) {
858 if (ecc_data2[0] == 0 &&
859 ecc_data2[1] == 0 &&
860 ecc_data2[2] == 0)
861 return 0;
862 }
863 pr_debug("UNCORRECTED_ERROR default\n");
864 return -EBADMSG;
865 }
866}
867
868/**
869 * omap_correct_data - Compares the ECC read with HW generated ECC
870 * @mtd: MTD device structure
871 * @dat: page data
872 * @read_ecc: ecc read from nand flash
873 * @calc_ecc: ecc read from HW ECC registers
874 *
875 * Compares the ecc read from nand spare area with ECC registers values
876 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error
877 * detection and correction. If there are no errors, %0 is returned. If
878 * there were errors and all of the errors were corrected, the number of
879 * corrected errors is returned. If uncorrectable errors exist, %-1 is
880 * returned.
881 */
882static int omap_correct_data(struct mtd_info *mtd, u_char *dat,
883 u_char *read_ecc, u_char *calc_ecc)
884{
885 struct omap_nand_info *info = mtd_to_omap(mtd);
886 int blockCnt = 0, i = 0, ret = 0;
887 int stat = 0;
888
889 /* Ex NAND_ECC_HW12_2048 */
890 if ((info->nand.ecc.mode == NAND_ECC_HW) &&
891 (info->nand.ecc.size == 2048))
892 blockCnt = 4;
893 else
894 blockCnt = 1;
895
896 for (i = 0; i < blockCnt; i++) {
897 if (memcmp(read_ecc, calc_ecc, 3) != 0) {
898 ret = omap_compare_ecc(read_ecc, calc_ecc, dat);
899 if (ret < 0)
900 return ret;
901 /* keep track of the number of corrected errors */
902 stat += ret;
903 }
904 read_ecc += 3;
905 calc_ecc += 3;
906 dat += 512;
907 }
908 return stat;
909}
910
911/**
912 * omap_calcuate_ecc - Generate non-inverted ECC bytes.
913 * @mtd: MTD device structure
914 * @dat: The pointer to data on which ecc is computed
915 * @ecc_code: The ecc_code buffer
916 *
917 * Using noninverted ECC can be considered ugly since writing a blank
918 * page ie. padding will clear the ECC bytes. This is no problem as long
919 * nobody is trying to write data on the seemingly unused page. Reading
920 * an erased page will produce an ECC mismatch between generated and read
921 * ECC bytes that has to be dealt with separately.
922 */
923static int omap_calculate_ecc(struct mtd_info *mtd, const u_char *dat,
924 u_char *ecc_code)
925{
926 struct omap_nand_info *info = mtd_to_omap(mtd);
927 u32 val;
928
929 val = readl(info->reg.gpmc_ecc_config);
930 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs)
931 return -EINVAL;
932
933 /* read ecc result */
934 val = readl(info->reg.gpmc_ecc1_result);
935 *ecc_code++ = val; /* P128e, ..., P1e */
936 *ecc_code++ = val >> 16; /* P128o, ..., P1o */
937 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */
938 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0);
939
940 return 0;
941}
942
943/**
944 * omap_enable_hwecc - This function enables the hardware ecc functionality
945 * @mtd: MTD device structure
946 * @mode: Read/Write mode
947 */
948static void omap_enable_hwecc(struct mtd_info *mtd, int mode)
949{
950 struct omap_nand_info *info = mtd_to_omap(mtd);
951 struct nand_chip *chip = mtd_to_nand(mtd);
952 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
953 u32 val;
954
955 /* clear ecc and enable bits */
956 val = ECCCLEAR | ECC1;
957 writel(val, info->reg.gpmc_ecc_control);
958
959 /* program ecc and result sizes */
960 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |
961 ECC1RESULTSIZE);
962 writel(val, info->reg.gpmc_ecc_size_config);
963
964 switch (mode) {
965 case NAND_ECC_READ:
966 case NAND_ECC_WRITE:
967 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
968 break;
969 case NAND_ECC_READSYN:
970 writel(ECCCLEAR, info->reg.gpmc_ecc_control);
971 break;
972 default:
973 dev_info(&info->pdev->dev,
974 "error: unrecognized Mode[%d]!\n", mode);
975 break;
976 }
977
978 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */
979 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1);
980 writel(val, info->reg.gpmc_ecc_config);
981}
982
983/**
984 * omap_wait - wait until the command is done
985 * @mtd: MTD device structure
986 * @chip: NAND Chip structure
987 *
988 * Wait function is called during Program and erase operations and
989 * the way it is called from MTD layer, we should wait till the NAND
990 * chip is ready after the programming/erase operation has completed.
991 *
992 * Erase can take up to 400ms and program up to 20ms according to
993 * general NAND and SmartMedia specs
994 */
995static int omap_wait(struct mtd_info *mtd, struct nand_chip *chip)
996{
997 struct nand_chip *this = mtd_to_nand(mtd);
998 struct omap_nand_info *info = mtd_to_omap(mtd);
999 unsigned long timeo = jiffies;
1000 int status, state = this->state;
1001
1002 if (state == FL_ERASING)
1003 timeo += msecs_to_jiffies(400);
1004 else
1005 timeo += msecs_to_jiffies(20);
1006
1007 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command);
1008 while (time_before(jiffies, timeo)) {
1009 status = readb(info->reg.gpmc_nand_data);
1010 if (status & NAND_STATUS_READY)
1011 break;
1012 cond_resched();
1013 }
1014
1015 status = readb(info->reg.gpmc_nand_data);
1016 return status;
1017}
1018
1019/**
1020 * omap_dev_ready - calls the platform specific dev_ready function
1021 * @mtd: MTD device structure
1022 */
1023static int omap_dev_ready(struct mtd_info *mtd)
1024{
1025 unsigned int val = 0;
1026 struct omap_nand_info *info = mtd_to_omap(mtd);
1027
1028 val = readl(info->reg.gpmc_status);
1029
1030 if ((val & 0x100) == 0x100) {
1031 return 1;
1032 } else {
1033 return 0;
1034 }
1035}
1036
1037/**
1038 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
1039 * @mtd: MTD device structure
1040 * @mode: Read/Write mode
1041 *
1042 * When using BCH with SW correction (i.e. no ELM), sector size is set
1043 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode
1044 * for both reading and writing with:
1045 * eccsize0 = 0 (no additional protected byte in spare area)
1046 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area)
1047 */
1048static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd, int mode)
1049{
1050 unsigned int bch_type;
1051 unsigned int dev_width, nsectors;
1052 struct omap_nand_info *info = mtd_to_omap(mtd);
1053 enum omap_ecc ecc_opt = info->ecc_opt;
1054 struct nand_chip *chip = mtd_to_nand(mtd);
1055 u32 val, wr_mode;
1056 unsigned int ecc_size1, ecc_size0;
1057
1058 /* GPMC configurations for calculating ECC */
1059 switch (ecc_opt) {
1060 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1061 bch_type = 0;
1062 nsectors = 1;
1063 wr_mode = BCH_WRAPMODE_6;
1064 ecc_size0 = BCH_ECC_SIZE0;
1065 ecc_size1 = BCH_ECC_SIZE1;
1066 break;
1067 case OMAP_ECC_BCH4_CODE_HW:
1068 bch_type = 0;
1069 nsectors = chip->ecc.steps;
1070 if (mode == NAND_ECC_READ) {
1071 wr_mode = BCH_WRAPMODE_1;
1072 ecc_size0 = BCH4R_ECC_SIZE0;
1073 ecc_size1 = BCH4R_ECC_SIZE1;
1074 } else {
1075 wr_mode = BCH_WRAPMODE_6;
1076 ecc_size0 = BCH_ECC_SIZE0;
1077 ecc_size1 = BCH_ECC_SIZE1;
1078 }
1079 break;
1080 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1081 bch_type = 1;
1082 nsectors = 1;
1083 wr_mode = BCH_WRAPMODE_6;
1084 ecc_size0 = BCH_ECC_SIZE0;
1085 ecc_size1 = BCH_ECC_SIZE1;
1086 break;
1087 case OMAP_ECC_BCH8_CODE_HW:
1088 bch_type = 1;
1089 nsectors = chip->ecc.steps;
1090 if (mode == NAND_ECC_READ) {
1091 wr_mode = BCH_WRAPMODE_1;
1092 ecc_size0 = BCH8R_ECC_SIZE0;
1093 ecc_size1 = BCH8R_ECC_SIZE1;
1094 } else {
1095 wr_mode = BCH_WRAPMODE_6;
1096 ecc_size0 = BCH_ECC_SIZE0;
1097 ecc_size1 = BCH_ECC_SIZE1;
1098 }
1099 break;
1100 case OMAP_ECC_BCH16_CODE_HW:
1101 bch_type = 0x2;
1102 nsectors = chip->ecc.steps;
1103 if (mode == NAND_ECC_READ) {
1104 wr_mode = 0x01;
1105 ecc_size0 = 52; /* ECC bits in nibbles per sector */
1106 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */
1107 } else {
1108 wr_mode = 0x01;
1109 ecc_size0 = 0; /* extra bits in nibbles per sector */
1110 ecc_size1 = 52; /* OOB bits in nibbles per sector */
1111 }
1112 break;
1113 default:
1114 return;
1115 }
1116
1117 writel(ECC1, info->reg.gpmc_ecc_control);
1118
1119 /* Configure ecc size for BCH */
1120 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT);
1121 writel(val, info->reg.gpmc_ecc_size_config);
1122
1123 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
1124
1125 /* BCH configuration */
1126 val = ((1 << 16) | /* enable BCH */
1127 (bch_type << 12) | /* BCH4/BCH8/BCH16 */
1128 (wr_mode << 8) | /* wrap mode */
1129 (dev_width << 7) | /* bus width */
1130 (((nsectors-1) & 0x7) << 4) | /* number of sectors */
1131 (info->gpmc_cs << 1) | /* ECC CS */
1132 (0x1)); /* enable ECC */
1133
1134 writel(val, info->reg.gpmc_ecc_config);
1135
1136 /* Clear ecc and enable bits */
1137 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
1138}
1139
1140static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f};
1141static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
1142 0x97, 0x79, 0xe5, 0x24, 0xb5};
1143
1144/**
1145 * omap_calculate_ecc_bch - Generate bytes of ECC bytes
1146 * @mtd: MTD device structure
1147 * @dat: The pointer to data on which ecc is computed
1148 * @ecc_code: The ecc_code buffer
1149 *
1150 * Support calculating of BCH4/8 ecc vectors for the page
1151 */
1152static int __maybe_unused omap_calculate_ecc_bch(struct mtd_info *mtd,
1153 const u_char *dat, u_char *ecc_calc)
1154{
1155 struct omap_nand_info *info = mtd_to_omap(mtd);
1156 int eccbytes = info->nand.ecc.bytes;
1157 struct gpmc_nand_regs *gpmc_regs = &info->reg;
1158 u8 *ecc_code;
1159 unsigned long nsectors, bch_val1, bch_val2, bch_val3, bch_val4;
1160 u32 val;
1161 int i, j;
1162
1163 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1;
1164 for (i = 0; i < nsectors; i++) {
1165 ecc_code = ecc_calc;
1166 switch (info->ecc_opt) {
1167 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1168 case OMAP_ECC_BCH8_CODE_HW:
1169 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1170 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1171 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]);
1172 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]);
1173 *ecc_code++ = (bch_val4 & 0xFF);
1174 *ecc_code++ = ((bch_val3 >> 24) & 0xFF);
1175 *ecc_code++ = ((bch_val3 >> 16) & 0xFF);
1176 *ecc_code++ = ((bch_val3 >> 8) & 0xFF);
1177 *ecc_code++ = (bch_val3 & 0xFF);
1178 *ecc_code++ = ((bch_val2 >> 24) & 0xFF);
1179 *ecc_code++ = ((bch_val2 >> 16) & 0xFF);
1180 *ecc_code++ = ((bch_val2 >> 8) & 0xFF);
1181 *ecc_code++ = (bch_val2 & 0xFF);
1182 *ecc_code++ = ((bch_val1 >> 24) & 0xFF);
1183 *ecc_code++ = ((bch_val1 >> 16) & 0xFF);
1184 *ecc_code++ = ((bch_val1 >> 8) & 0xFF);
1185 *ecc_code++ = (bch_val1 & 0xFF);
1186 break;
1187 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1188 case OMAP_ECC_BCH4_CODE_HW:
1189 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1190 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1191 *ecc_code++ = ((bch_val2 >> 12) & 0xFF);
1192 *ecc_code++ = ((bch_val2 >> 4) & 0xFF);
1193 *ecc_code++ = ((bch_val2 & 0xF) << 4) |
1194 ((bch_val1 >> 28) & 0xF);
1195 *ecc_code++ = ((bch_val1 >> 20) & 0xFF);
1196 *ecc_code++ = ((bch_val1 >> 12) & 0xFF);
1197 *ecc_code++ = ((bch_val1 >> 4) & 0xFF);
1198 *ecc_code++ = ((bch_val1 & 0xF) << 4);
1199 break;
1200 case OMAP_ECC_BCH16_CODE_HW:
1201 val = readl(gpmc_regs->gpmc_bch_result6[i]);
1202 ecc_code[0] = ((val >> 8) & 0xFF);
1203 ecc_code[1] = ((val >> 0) & 0xFF);
1204 val = readl(gpmc_regs->gpmc_bch_result5[i]);
1205 ecc_code[2] = ((val >> 24) & 0xFF);
1206 ecc_code[3] = ((val >> 16) & 0xFF);
1207 ecc_code[4] = ((val >> 8) & 0xFF);
1208 ecc_code[5] = ((val >> 0) & 0xFF);
1209 val = readl(gpmc_regs->gpmc_bch_result4[i]);
1210 ecc_code[6] = ((val >> 24) & 0xFF);
1211 ecc_code[7] = ((val >> 16) & 0xFF);
1212 ecc_code[8] = ((val >> 8) & 0xFF);
1213 ecc_code[9] = ((val >> 0) & 0xFF);
1214 val = readl(gpmc_regs->gpmc_bch_result3[i]);
1215 ecc_code[10] = ((val >> 24) & 0xFF);
1216 ecc_code[11] = ((val >> 16) & 0xFF);
1217 ecc_code[12] = ((val >> 8) & 0xFF);
1218 ecc_code[13] = ((val >> 0) & 0xFF);
1219 val = readl(gpmc_regs->gpmc_bch_result2[i]);
1220 ecc_code[14] = ((val >> 24) & 0xFF);
1221 ecc_code[15] = ((val >> 16) & 0xFF);
1222 ecc_code[16] = ((val >> 8) & 0xFF);
1223 ecc_code[17] = ((val >> 0) & 0xFF);
1224 val = readl(gpmc_regs->gpmc_bch_result1[i]);
1225 ecc_code[18] = ((val >> 24) & 0xFF);
1226 ecc_code[19] = ((val >> 16) & 0xFF);
1227 ecc_code[20] = ((val >> 8) & 0xFF);
1228 ecc_code[21] = ((val >> 0) & 0xFF);
1229 val = readl(gpmc_regs->gpmc_bch_result0[i]);
1230 ecc_code[22] = ((val >> 24) & 0xFF);
1231 ecc_code[23] = ((val >> 16) & 0xFF);
1232 ecc_code[24] = ((val >> 8) & 0xFF);
1233 ecc_code[25] = ((val >> 0) & 0xFF);
1234 break;
1235 default:
1236 return -EINVAL;
1237 }
1238
1239 /* ECC scheme specific syndrome customizations */
1240 switch (info->ecc_opt) {
1241 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1242 /* Add constant polynomial to remainder, so that
1243 * ECC of blank pages results in 0x0 on reading back */
1244 for (j = 0; j < eccbytes; j++)
1245 ecc_calc[j] ^= bch4_polynomial[j];
1246 break;
1247 case OMAP_ECC_BCH4_CODE_HW:
1248 /* Set 8th ECC byte as 0x0 for ROM compatibility */
1249 ecc_calc[eccbytes - 1] = 0x0;
1250 break;
1251 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1252 /* Add constant polynomial to remainder, so that
1253 * ECC of blank pages results in 0x0 on reading back */
1254 for (j = 0; j < eccbytes; j++)
1255 ecc_calc[j] ^= bch8_polynomial[j];
1256 break;
1257 case OMAP_ECC_BCH8_CODE_HW:
1258 /* Set 14th ECC byte as 0x0 for ROM compatibility */
1259 ecc_calc[eccbytes - 1] = 0x0;
1260 break;
1261 case OMAP_ECC_BCH16_CODE_HW:
1262 break;
1263 default:
1264 return -EINVAL;
1265 }
1266
1267 ecc_calc += eccbytes;
1268 }
1269
1270 return 0;
1271}
1272
1273/**
1274 * erased_sector_bitflips - count bit flips
1275 * @data: data sector buffer
1276 * @oob: oob buffer
1277 * @info: omap_nand_info
1278 *
1279 * Check the bit flips in erased page falls below correctable level.
1280 * If falls below, report the page as erased with correctable bit
1281 * flip, else report as uncorrectable page.
1282 */
1283static int erased_sector_bitflips(u_char *data, u_char *oob,
1284 struct omap_nand_info *info)
1285{
1286 int flip_bits = 0, i;
1287
1288 for (i = 0; i < info->nand.ecc.size; i++) {
1289 flip_bits += hweight8(~data[i]);
1290 if (flip_bits > info->nand.ecc.strength)
1291 return 0;
1292 }
1293
1294 for (i = 0; i < info->nand.ecc.bytes - 1; i++) {
1295 flip_bits += hweight8(~oob[i]);
1296 if (flip_bits > info->nand.ecc.strength)
1297 return 0;
1298 }
1299
1300 /*
1301 * Bit flips falls in correctable level.
1302 * Fill data area with 0xFF
1303 */
1304 if (flip_bits) {
1305 memset(data, 0xFF, info->nand.ecc.size);
1306 memset(oob, 0xFF, info->nand.ecc.bytes);
1307 }
1308
1309 return flip_bits;
1310}
1311
1312/**
1313 * omap_elm_correct_data - corrects page data area in case error reported
1314 * @mtd: MTD device structure
1315 * @data: page data
1316 * @read_ecc: ecc read from nand flash
1317 * @calc_ecc: ecc read from HW ECC registers
1318 *
1319 * Calculated ecc vector reported as zero in case of non-error pages.
1320 * In case of non-zero ecc vector, first filter out erased-pages, and
1321 * then process data via ELM to detect bit-flips.
1322 */
1323static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,
1324 u_char *read_ecc, u_char *calc_ecc)
1325{
1326 struct omap_nand_info *info = mtd_to_omap(mtd);
1327 struct nand_ecc_ctrl *ecc = &info->nand.ecc;
1328 int eccsteps = info->nand.ecc.steps;
1329 int i , j, stat = 0;
1330 int eccflag, actual_eccbytes;
1331 struct elm_errorvec err_vec[ERROR_VECTOR_MAX];
1332 u_char *ecc_vec = calc_ecc;
1333 u_char *spare_ecc = read_ecc;
1334 u_char *erased_ecc_vec;
1335 u_char *buf;
1336 int bitflip_count;
1337 bool is_error_reported = false;
1338 u32 bit_pos, byte_pos, error_max, pos;
1339 int err;
1340
1341 switch (info->ecc_opt) {
1342 case OMAP_ECC_BCH4_CODE_HW:
1343 /* omit 7th ECC byte reserved for ROM code compatibility */
1344 actual_eccbytes = ecc->bytes - 1;
1345 erased_ecc_vec = bch4_vector;
1346 break;
1347 case OMAP_ECC_BCH8_CODE_HW:
1348 /* omit 14th ECC byte reserved for ROM code compatibility */
1349 actual_eccbytes = ecc->bytes - 1;
1350 erased_ecc_vec = bch8_vector;
1351 break;
1352 case OMAP_ECC_BCH16_CODE_HW:
1353 actual_eccbytes = ecc->bytes;
1354 erased_ecc_vec = bch16_vector;
1355 break;
1356 default:
1357 dev_err(&info->pdev->dev, "invalid driver configuration\n");
1358 return -EINVAL;
1359 }
1360
1361 /* Initialize elm error vector to zero */
1362 memset(err_vec, 0, sizeof(err_vec));
1363
1364 for (i = 0; i < eccsteps ; i++) {
1365 eccflag = 0; /* initialize eccflag */
1366
1367 /*
1368 * Check any error reported,
1369 * In case of error, non zero ecc reported.
1370 */
1371 for (j = 0; j < actual_eccbytes; j++) {
1372 if (calc_ecc[j] != 0) {
1373 eccflag = 1; /* non zero ecc, error present */
1374 break;
1375 }
1376 }
1377
1378 if (eccflag == 1) {
1379 if (memcmp(calc_ecc, erased_ecc_vec,
1380 actual_eccbytes) == 0) {
1381 /*
1382 * calc_ecc[] matches pattern for ECC(all 0xff)
1383 * so this is definitely an erased-page
1384 */
1385 } else {
1386 buf = &data[info->nand.ecc.size * i];
1387 /*
1388 * count number of 0-bits in read_buf.
1389 * This check can be removed once a similar
1390 * check is introduced in generic NAND driver
1391 */
1392 bitflip_count = erased_sector_bitflips(
1393 buf, read_ecc, info);
1394 if (bitflip_count) {
1395 /*
1396 * number of 0-bits within ECC limits
1397 * So this may be an erased-page
1398 */
1399 stat += bitflip_count;
1400 } else {
1401 /*
1402 * Too many 0-bits. It may be a
1403 * - programmed-page, OR
1404 * - erased-page with many bit-flips
1405 * So this page requires check by ELM
1406 */
1407 err_vec[i].error_reported = true;
1408 is_error_reported = true;
1409 }
1410 }
1411 }
1412
1413 /* Update the ecc vector */
1414 calc_ecc += ecc->bytes;
1415 read_ecc += ecc->bytes;
1416 }
1417
1418 /* Check if any error reported */
1419 if (!is_error_reported)
1420 return stat;
1421
1422 /* Decode BCH error using ELM module */
1423 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec);
1424
1425 err = 0;
1426 for (i = 0; i < eccsteps; i++) {
1427 if (err_vec[i].error_uncorrectable) {
1428 dev_err(&info->pdev->dev,
1429 "uncorrectable bit-flips found\n");
1430 err = -EBADMSG;
1431 } else if (err_vec[i].error_reported) {
1432 for (j = 0; j < err_vec[i].error_count; j++) {
1433 switch (info->ecc_opt) {
1434 case OMAP_ECC_BCH4_CODE_HW:
1435 /* Add 4 bits to take care of padding */
1436 pos = err_vec[i].error_loc[j] +
1437 BCH4_BIT_PAD;
1438 break;
1439 case OMAP_ECC_BCH8_CODE_HW:
1440 case OMAP_ECC_BCH16_CODE_HW:
1441 pos = err_vec[i].error_loc[j];
1442 break;
1443 default:
1444 return -EINVAL;
1445 }
1446 error_max = (ecc->size + actual_eccbytes) * 8;
1447 /* Calculate bit position of error */
1448 bit_pos = pos % 8;
1449
1450 /* Calculate byte position of error */
1451 byte_pos = (error_max - pos - 1) / 8;
1452
1453 if (pos < error_max) {
1454 if (byte_pos < 512) {
1455 pr_debug("bitflip@dat[%d]=%x\n",
1456 byte_pos, data[byte_pos]);
1457 data[byte_pos] ^= 1 << bit_pos;
1458 } else {
1459 pr_debug("bitflip@oob[%d]=%x\n",
1460 (byte_pos - 512),
1461 spare_ecc[byte_pos - 512]);
1462 spare_ecc[byte_pos - 512] ^=
1463 1 << bit_pos;
1464 }
1465 } else {
1466 dev_err(&info->pdev->dev,
1467 "invalid bit-flip @ %d:%d\n",
1468 byte_pos, bit_pos);
1469 err = -EBADMSG;
1470 }
1471 }
1472 }
1473
1474 /* Update number of correctable errors */
1475 stat += err_vec[i].error_count;
1476
1477 /* Update page data with sector size */
1478 data += ecc->size;
1479 spare_ecc += ecc->bytes;
1480 }
1481
1482 return (err) ? err : stat;
1483}
1484
1485/**
1486 * omap_write_page_bch - BCH ecc based write page function for entire page
1487 * @mtd: mtd info structure
1488 * @chip: nand chip info structure
1489 * @buf: data buffer
1490 * @oob_required: must write chip->oob_poi to OOB
1491 * @page: page
1492 *
1493 * Custom write page method evolved to support multi sector writing in one shot
1494 */
1495static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
1496 const uint8_t *buf, int oob_required, int page)
1497{
1498 int i;
1499 uint8_t *ecc_calc = chip->buffers->ecccalc;
1500 uint32_t *eccpos = chip->ecc.layout->eccpos;
1501
1502 /* Enable GPMC ecc engine */
1503 chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
1504
1505 /* Write data */
1506 chip->write_buf(mtd, buf, mtd->writesize);
1507
1508 /* Update ecc vector from GPMC result registers */
1509 chip->ecc.calculate(mtd, buf, &ecc_calc[0]);
1510
1511 for (i = 0; i < chip->ecc.total; i++)
1512 chip->oob_poi[eccpos[i]] = ecc_calc[i];
1513
1514 /* Write ecc vector to OOB area */
1515 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1516 return 0;
1517}
1518
1519/**
1520 * omap_read_page_bch - BCH ecc based page read function for entire page
1521 * @mtd: mtd info structure
1522 * @chip: nand chip info structure
1523 * @buf: buffer to store read data
1524 * @oob_required: caller requires OOB data read to chip->oob_poi
1525 * @page: page number to read
1526 *
1527 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module
1528 * used for error correction.
1529 * Custom method evolved to support ELM error correction & multi sector
1530 * reading. On reading page data area is read along with OOB data with
1531 * ecc engine enabled. ecc vector updated after read of OOB data.
1532 * For non error pages ecc vector reported as zero.
1533 */
1534static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
1535 uint8_t *buf, int oob_required, int page)
1536{
1537 uint8_t *ecc_calc = chip->buffers->ecccalc;
1538 uint8_t *ecc_code = chip->buffers->ecccode;
1539 uint32_t *eccpos = chip->ecc.layout->eccpos;
1540 uint8_t *oob = &chip->oob_poi[eccpos[0]];
1541 uint32_t oob_pos = mtd->writesize + chip->ecc.layout->eccpos[0];
1542 int stat;
1543 unsigned int max_bitflips = 0;
1544
1545 /* Enable GPMC ecc engine */
1546 chip->ecc.hwctl(mtd, NAND_ECC_READ);
1547
1548 /* Read data */
1549 chip->read_buf(mtd, buf, mtd->writesize);
1550
1551 /* Read oob bytes */
1552 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
1553 chip->read_buf(mtd, oob, chip->ecc.total);
1554
1555 /* Calculate ecc bytes */
1556 chip->ecc.calculate(mtd, buf, ecc_calc);
1557
1558 memcpy(ecc_code, &chip->oob_poi[eccpos[0]], chip->ecc.total);
1559
1560 stat = chip->ecc.correct(mtd, buf, ecc_code, ecc_calc);
1561
1562 if (stat < 0) {
1563 mtd->ecc_stats.failed++;
1564 } else {
1565 mtd->ecc_stats.corrected += stat;
1566 max_bitflips = max_t(unsigned int, max_bitflips, stat);
1567 }
1568
1569 return max_bitflips;
1570}
1571
1572/**
1573 * is_elm_present - checks for presence of ELM module by scanning DT nodes
1574 * @omap_nand_info: NAND device structure containing platform data
1575 */
1576static bool is_elm_present(struct omap_nand_info *info,
1577 struct device_node *elm_node)
1578{
1579 struct platform_device *pdev;
1580
1581 /* check whether elm-id is passed via DT */
1582 if (!elm_node) {
1583 dev_err(&info->pdev->dev, "ELM devicetree node not found\n");
1584 return false;
1585 }
1586 pdev = of_find_device_by_node(elm_node);
1587 /* check whether ELM device is registered */
1588 if (!pdev) {
1589 dev_err(&info->pdev->dev, "ELM device not found\n");
1590 return false;
1591 }
1592 /* ELM module available, now configure it */
1593 info->elm_dev = &pdev->dev;
1594 return true;
1595}
1596
1597static bool omap2_nand_ecc_check(struct omap_nand_info *info,
1598 struct omap_nand_platform_data *pdata)
1599{
1600 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm;
1601
1602 switch (info->ecc_opt) {
1603 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1604 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1605 ecc_needs_omap_bch = false;
1606 ecc_needs_bch = true;
1607 ecc_needs_elm = false;
1608 break;
1609 case OMAP_ECC_BCH4_CODE_HW:
1610 case OMAP_ECC_BCH8_CODE_HW:
1611 case OMAP_ECC_BCH16_CODE_HW:
1612 ecc_needs_omap_bch = true;
1613 ecc_needs_bch = false;
1614 ecc_needs_elm = true;
1615 break;
1616 default:
1617 ecc_needs_omap_bch = false;
1618 ecc_needs_bch = false;
1619 ecc_needs_elm = false;
1620 break;
1621 }
1622
1623 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_BCH)) {
1624 dev_err(&info->pdev->dev,
1625 "CONFIG_MTD_NAND_ECC_BCH not enabled\n");
1626 return false;
1627 }
1628 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) {
1629 dev_err(&info->pdev->dev,
1630 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n");
1631 return false;
1632 }
1633 if (ecc_needs_elm && !is_elm_present(info, pdata->elm_of_node)) {
1634 dev_err(&info->pdev->dev, "ELM not available\n");
1635 return false;
1636 }
1637
1638 return true;
1639}
1640
1641static int omap_nand_probe(struct platform_device *pdev)
1642{
1643 struct omap_nand_info *info;
1644 struct omap_nand_platform_data *pdata;
1645 struct mtd_info *mtd;
1646 struct nand_chip *nand_chip;
1647 struct nand_ecclayout *ecclayout;
1648 int err;
1649 int i;
1650 dma_cap_mask_t mask;
1651 unsigned sig;
1652 unsigned oob_index;
1653 struct resource *res;
1654
1655 pdata = dev_get_platdata(&pdev->dev);
1656 if (pdata == NULL) {
1657 dev_err(&pdev->dev, "platform data missing\n");
1658 return -ENODEV;
1659 }
1660
1661 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info),
1662 GFP_KERNEL);
1663 if (!info)
1664 return -ENOMEM;
1665
1666 platform_set_drvdata(pdev, info);
1667
1668 info->pdev = pdev;
1669 info->gpmc_cs = pdata->cs;
1670 info->reg = pdata->reg;
1671 info->of_node = pdata->of_node;
1672 info->ecc_opt = pdata->ecc_opt;
1673 nand_chip = &info->nand;
1674 mtd = nand_to_mtd(nand_chip);
1675 mtd->dev.parent = &pdev->dev;
1676 nand_chip->ecc.priv = NULL;
1677 nand_set_flash_node(nand_chip, pdata->of_node);
1678
1679 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1680 nand_chip->IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res);
1681 if (IS_ERR(nand_chip->IO_ADDR_R))
1682 return PTR_ERR(nand_chip->IO_ADDR_R);
1683
1684 info->phys_base = res->start;
1685
1686 nand_chip->controller = &omap_gpmc_controller;
1687
1688 nand_chip->IO_ADDR_W = nand_chip->IO_ADDR_R;
1689 nand_chip->cmd_ctrl = omap_hwcontrol;
1690
1691 /*
1692 * If RDY/BSY line is connected to OMAP then use the omap ready
1693 * function and the generic nand_wait function which reads the status
1694 * register after monitoring the RDY/BSY line. Otherwise use a standard
1695 * chip delay which is slightly more than tR (AC Timing) of the NAND
1696 * device and read status register until you get a failure or success
1697 */
1698 if (pdata->dev_ready) {
1699 nand_chip->dev_ready = omap_dev_ready;
1700 nand_chip->chip_delay = 0;
1701 } else {
1702 nand_chip->waitfunc = omap_wait;
1703 nand_chip->chip_delay = 50;
1704 }
1705
1706 if (pdata->flash_bbt)
1707 nand_chip->bbt_options |= NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1708 else
1709 nand_chip->options |= NAND_SKIP_BBTSCAN;
1710
1711 /* scan NAND device connected to chip controller */
1712 nand_chip->options |= pdata->devsize & NAND_BUSWIDTH_16;
1713 if (nand_scan_ident(mtd, 1, NULL)) {
1714 dev_err(&info->pdev->dev, "scan failed, may be bus-width mismatch\n");
1715 err = -ENXIO;
1716 goto return_error;
1717 }
1718
1719 /* re-populate low-level callbacks based on xfer modes */
1720 switch (pdata->xfer_type) {
1721 case NAND_OMAP_PREFETCH_POLLED:
1722 nand_chip->read_buf = omap_read_buf_pref;
1723 nand_chip->write_buf = omap_write_buf_pref;
1724 break;
1725
1726 case NAND_OMAP_POLLED:
1727 /* Use nand_base defaults for {read,write}_buf */
1728 break;
1729
1730 case NAND_OMAP_PREFETCH_DMA:
1731 dma_cap_zero(mask);
1732 dma_cap_set(DMA_SLAVE, mask);
1733 sig = OMAP24XX_DMA_GPMC;
1734 info->dma = dma_request_channel(mask, omap_dma_filter_fn, &sig);
1735 if (!info->dma) {
1736 dev_err(&pdev->dev, "DMA engine request failed\n");
1737 err = -ENXIO;
1738 goto return_error;
1739 } else {
1740 struct dma_slave_config cfg;
1741
1742 memset(&cfg, 0, sizeof(cfg));
1743 cfg.src_addr = info->phys_base;
1744 cfg.dst_addr = info->phys_base;
1745 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1746 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1747 cfg.src_maxburst = 16;
1748 cfg.dst_maxburst = 16;
1749 err = dmaengine_slave_config(info->dma, &cfg);
1750 if (err) {
1751 dev_err(&pdev->dev, "DMA engine slave config failed: %d\n",
1752 err);
1753 goto return_error;
1754 }
1755 nand_chip->read_buf = omap_read_buf_dma_pref;
1756 nand_chip->write_buf = omap_write_buf_dma_pref;
1757 }
1758 break;
1759
1760 case NAND_OMAP_PREFETCH_IRQ:
1761 info->gpmc_irq_fifo = platform_get_irq(pdev, 0);
1762 if (info->gpmc_irq_fifo <= 0) {
1763 dev_err(&pdev->dev, "error getting fifo irq\n");
1764 err = -ENODEV;
1765 goto return_error;
1766 }
1767 err = devm_request_irq(&pdev->dev, info->gpmc_irq_fifo,
1768 omap_nand_irq, IRQF_SHARED,
1769 "gpmc-nand-fifo", info);
1770 if (err) {
1771 dev_err(&pdev->dev, "requesting irq(%d) error:%d",
1772 info->gpmc_irq_fifo, err);
1773 info->gpmc_irq_fifo = 0;
1774 goto return_error;
1775 }
1776
1777 info->gpmc_irq_count = platform_get_irq(pdev, 1);
1778 if (info->gpmc_irq_count <= 0) {
1779 dev_err(&pdev->dev, "error getting count irq\n");
1780 err = -ENODEV;
1781 goto return_error;
1782 }
1783 err = devm_request_irq(&pdev->dev, info->gpmc_irq_count,
1784 omap_nand_irq, IRQF_SHARED,
1785 "gpmc-nand-count", info);
1786 if (err) {
1787 dev_err(&pdev->dev, "requesting irq(%d) error:%d",
1788 info->gpmc_irq_count, err);
1789 info->gpmc_irq_count = 0;
1790 goto return_error;
1791 }
1792
1793 nand_chip->read_buf = omap_read_buf_irq_pref;
1794 nand_chip->write_buf = omap_write_buf_irq_pref;
1795
1796 break;
1797
1798 default:
1799 dev_err(&pdev->dev,
1800 "xfer_type(%d) not supported!\n", pdata->xfer_type);
1801 err = -EINVAL;
1802 goto return_error;
1803 }
1804
1805 if (!omap2_nand_ecc_check(info, pdata)) {
1806 err = -EINVAL;
1807 goto return_error;
1808 }
1809
1810 /*
1811 * Bail out earlier to let NAND_ECC_SOFT code create its own
1812 * ecclayout instead of using ours.
1813 */
1814 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) {
1815 nand_chip->ecc.mode = NAND_ECC_SOFT;
1816 goto scan_tail;
1817 }
1818
1819 /* populate MTD interface based on ECC scheme */
1820 ecclayout = &info->oobinfo;
1821 nand_chip->ecc.layout = ecclayout;
1822 switch (info->ecc_opt) {
1823 case OMAP_ECC_HAM1_CODE_HW:
1824 pr_info("nand: using OMAP_ECC_HAM1_CODE_HW\n");
1825 nand_chip->ecc.mode = NAND_ECC_HW;
1826 nand_chip->ecc.bytes = 3;
1827 nand_chip->ecc.size = 512;
1828 nand_chip->ecc.strength = 1;
1829 nand_chip->ecc.calculate = omap_calculate_ecc;
1830 nand_chip->ecc.hwctl = omap_enable_hwecc;
1831 nand_chip->ecc.correct = omap_correct_data;
1832 /* define ECC layout */
1833 ecclayout->eccbytes = nand_chip->ecc.bytes *
1834 (mtd->writesize /
1835 nand_chip->ecc.size);
1836 if (nand_chip->options & NAND_BUSWIDTH_16)
1837 oob_index = BADBLOCK_MARKER_LENGTH;
1838 else
1839 oob_index = 1;
1840 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++)
1841 ecclayout->eccpos[i] = oob_index;
1842 /* no reserved-marker in ecclayout for this ecc-scheme */
1843 ecclayout->oobfree->offset =
1844 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1845 break;
1846
1847 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1848 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n");
1849 nand_chip->ecc.mode = NAND_ECC_HW;
1850 nand_chip->ecc.size = 512;
1851 nand_chip->ecc.bytes = 7;
1852 nand_chip->ecc.strength = 4;
1853 nand_chip->ecc.hwctl = omap_enable_hwecc_bch;
1854 nand_chip->ecc.correct = nand_bch_correct_data;
1855 nand_chip->ecc.calculate = omap_calculate_ecc_bch;
1856 /* define ECC layout */
1857 ecclayout->eccbytes = nand_chip->ecc.bytes *
1858 (mtd->writesize /
1859 nand_chip->ecc.size);
1860 oob_index = BADBLOCK_MARKER_LENGTH;
1861 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++) {
1862 ecclayout->eccpos[i] = oob_index;
1863 if (((i + 1) % nand_chip->ecc.bytes) == 0)
1864 oob_index++;
1865 }
1866 /* include reserved-marker in ecclayout->oobfree calculation */
1867 ecclayout->oobfree->offset = 1 +
1868 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1869 /* software bch library is used for locating errors */
1870 nand_chip->ecc.priv = nand_bch_init(mtd);
1871 if (!nand_chip->ecc.priv) {
1872 dev_err(&info->pdev->dev, "unable to use BCH library\n");
1873 err = -EINVAL;
1874 goto return_error;
1875 }
1876 break;
1877
1878 case OMAP_ECC_BCH4_CODE_HW:
1879 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n");
1880 nand_chip->ecc.mode = NAND_ECC_HW;
1881 nand_chip->ecc.size = 512;
1882 /* 14th bit is kept reserved for ROM-code compatibility */
1883 nand_chip->ecc.bytes = 7 + 1;
1884 nand_chip->ecc.strength = 4;
1885 nand_chip->ecc.hwctl = omap_enable_hwecc_bch;
1886 nand_chip->ecc.correct = omap_elm_correct_data;
1887 nand_chip->ecc.calculate = omap_calculate_ecc_bch;
1888 nand_chip->ecc.read_page = omap_read_page_bch;
1889 nand_chip->ecc.write_page = omap_write_page_bch;
1890 /* define ECC layout */
1891 ecclayout->eccbytes = nand_chip->ecc.bytes *
1892 (mtd->writesize /
1893 nand_chip->ecc.size);
1894 oob_index = BADBLOCK_MARKER_LENGTH;
1895 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++)
1896 ecclayout->eccpos[i] = oob_index;
1897 /* reserved marker already included in ecclayout->eccbytes */
1898 ecclayout->oobfree->offset =
1899 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1900
1901 err = elm_config(info->elm_dev, BCH4_ECC,
1902 mtd->writesize / nand_chip->ecc.size,
1903 nand_chip->ecc.size, nand_chip->ecc.bytes);
1904 if (err < 0)
1905 goto return_error;
1906 break;
1907
1908 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1909 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
1910 nand_chip->ecc.mode = NAND_ECC_HW;
1911 nand_chip->ecc.size = 512;
1912 nand_chip->ecc.bytes = 13;
1913 nand_chip->ecc.strength = 8;
1914 nand_chip->ecc.hwctl = omap_enable_hwecc_bch;
1915 nand_chip->ecc.correct = nand_bch_correct_data;
1916 nand_chip->ecc.calculate = omap_calculate_ecc_bch;
1917 /* define ECC layout */
1918 ecclayout->eccbytes = nand_chip->ecc.bytes *
1919 (mtd->writesize /
1920 nand_chip->ecc.size);
1921 oob_index = BADBLOCK_MARKER_LENGTH;
1922 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++) {
1923 ecclayout->eccpos[i] = oob_index;
1924 if (((i + 1) % nand_chip->ecc.bytes) == 0)
1925 oob_index++;
1926 }
1927 /* include reserved-marker in ecclayout->oobfree calculation */
1928 ecclayout->oobfree->offset = 1 +
1929 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1930 /* software bch library is used for locating errors */
1931 nand_chip->ecc.priv = nand_bch_init(mtd);
1932 if (!nand_chip->ecc.priv) {
1933 dev_err(&info->pdev->dev, "unable to use BCH library\n");
1934 err = -EINVAL;
1935 goto return_error;
1936 }
1937 break;
1938
1939 case OMAP_ECC_BCH8_CODE_HW:
1940 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n");
1941 nand_chip->ecc.mode = NAND_ECC_HW;
1942 nand_chip->ecc.size = 512;
1943 /* 14th bit is kept reserved for ROM-code compatibility */
1944 nand_chip->ecc.bytes = 13 + 1;
1945 nand_chip->ecc.strength = 8;
1946 nand_chip->ecc.hwctl = omap_enable_hwecc_bch;
1947 nand_chip->ecc.correct = omap_elm_correct_data;
1948 nand_chip->ecc.calculate = omap_calculate_ecc_bch;
1949 nand_chip->ecc.read_page = omap_read_page_bch;
1950 nand_chip->ecc.write_page = omap_write_page_bch;
1951
1952 err = elm_config(info->elm_dev, BCH8_ECC,
1953 mtd->writesize / nand_chip->ecc.size,
1954 nand_chip->ecc.size, nand_chip->ecc.bytes);
1955 if (err < 0)
1956 goto return_error;
1957
1958 /* define ECC layout */
1959 ecclayout->eccbytes = nand_chip->ecc.bytes *
1960 (mtd->writesize /
1961 nand_chip->ecc.size);
1962 oob_index = BADBLOCK_MARKER_LENGTH;
1963 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++)
1964 ecclayout->eccpos[i] = oob_index;
1965 /* reserved marker already included in ecclayout->eccbytes */
1966 ecclayout->oobfree->offset =
1967 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1968 break;
1969
1970 case OMAP_ECC_BCH16_CODE_HW:
1971 pr_info("using OMAP_ECC_BCH16_CODE_HW ECC scheme\n");
1972 nand_chip->ecc.mode = NAND_ECC_HW;
1973 nand_chip->ecc.size = 512;
1974 nand_chip->ecc.bytes = 26;
1975 nand_chip->ecc.strength = 16;
1976 nand_chip->ecc.hwctl = omap_enable_hwecc_bch;
1977 nand_chip->ecc.correct = omap_elm_correct_data;
1978 nand_chip->ecc.calculate = omap_calculate_ecc_bch;
1979 nand_chip->ecc.read_page = omap_read_page_bch;
1980 nand_chip->ecc.write_page = omap_write_page_bch;
1981
1982 err = elm_config(info->elm_dev, BCH16_ECC,
1983 mtd->writesize / nand_chip->ecc.size,
1984 nand_chip->ecc.size, nand_chip->ecc.bytes);
1985 if (err < 0)
1986 goto return_error;
1987
1988 /* define ECC layout */
1989 ecclayout->eccbytes = nand_chip->ecc.bytes *
1990 (mtd->writesize /
1991 nand_chip->ecc.size);
1992 oob_index = BADBLOCK_MARKER_LENGTH;
1993 for (i = 0; i < ecclayout->eccbytes; i++, oob_index++)
1994 ecclayout->eccpos[i] = oob_index;
1995 /* reserved marker already included in ecclayout->eccbytes */
1996 ecclayout->oobfree->offset =
1997 ecclayout->eccpos[ecclayout->eccbytes - 1] + 1;
1998 break;
1999 default:
2000 dev_err(&info->pdev->dev, "invalid or unsupported ECC scheme\n");
2001 err = -EINVAL;
2002 goto return_error;
2003 }
2004
2005 /* all OOB bytes from oobfree->offset till end off OOB are free */
2006 ecclayout->oobfree->length = mtd->oobsize - ecclayout->oobfree->offset;
2007 /* check if NAND device's OOB is enough to store ECC signatures */
2008 if (mtd->oobsize < (ecclayout->eccbytes + BADBLOCK_MARKER_LENGTH)) {
2009 dev_err(&info->pdev->dev,
2010 "not enough OOB bytes required = %d, available=%d\n",
2011 ecclayout->eccbytes, mtd->oobsize);
2012 err = -EINVAL;
2013 goto return_error;
2014 }
2015
2016scan_tail:
2017 /* second phase scan */
2018 if (nand_scan_tail(mtd)) {
2019 err = -ENXIO;
2020 goto return_error;
2021 }
2022
2023 mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2024
2025 platform_set_drvdata(pdev, mtd);
2026
2027 return 0;
2028
2029return_error:
2030 if (info->dma)
2031 dma_release_channel(info->dma);
2032 if (nand_chip->ecc.priv) {
2033 nand_bch_free(nand_chip->ecc.priv);
2034 nand_chip->ecc.priv = NULL;
2035 }
2036 return err;
2037}
2038
2039static int omap_nand_remove(struct platform_device *pdev)
2040{
2041 struct mtd_info *mtd = platform_get_drvdata(pdev);
2042 struct nand_chip *nand_chip = mtd_to_nand(mtd);
2043 struct omap_nand_info *info = mtd_to_omap(mtd);
2044 if (nand_chip->ecc.priv) {
2045 nand_bch_free(nand_chip->ecc.priv);
2046 nand_chip->ecc.priv = NULL;
2047 }
2048 if (info->dma)
2049 dma_release_channel(info->dma);
2050 nand_release(mtd);
2051 return 0;
2052}
2053
2054static struct platform_driver omap_nand_driver = {
2055 .probe = omap_nand_probe,
2056 .remove = omap_nand_remove,
2057 .driver = {
2058 .name = DRIVER_NAME,
2059 },
2060};
2061
2062module_platform_driver(omap_nand_driver);
2063
2064MODULE_ALIAS("platform:" DRIVER_NAME);
2065MODULE_LICENSE("GPL");
2066MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards");