Loading...
Note: File does not exist in v6.8.
1/*
2 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
3 *
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 */
13
14#include <linux/clk.h>
15#include <linux/slab.h>
16#include <linux/bitops.h>
17#include <linux/dma-mapping.h>
18#include <linux/dmaengine.h>
19#include <linux/module.h>
20#include <linux/mtd/nand.h>
21#include <linux/mtd/partitions.h>
22#include <linux/of.h>
23#include <linux/of_device.h>
24#include <linux/delay.h>
25
26/* NANDc reg offsets */
27#define NAND_FLASH_CMD 0x00
28#define NAND_ADDR0 0x04
29#define NAND_ADDR1 0x08
30#define NAND_FLASH_CHIP_SELECT 0x0c
31#define NAND_EXEC_CMD 0x10
32#define NAND_FLASH_STATUS 0x14
33#define NAND_BUFFER_STATUS 0x18
34#define NAND_DEV0_CFG0 0x20
35#define NAND_DEV0_CFG1 0x24
36#define NAND_DEV0_ECC_CFG 0x28
37#define NAND_DEV1_ECC_CFG 0x2c
38#define NAND_DEV1_CFG0 0x30
39#define NAND_DEV1_CFG1 0x34
40#define NAND_READ_ID 0x40
41#define NAND_READ_STATUS 0x44
42#define NAND_DEV_CMD0 0xa0
43#define NAND_DEV_CMD1 0xa4
44#define NAND_DEV_CMD2 0xa8
45#define NAND_DEV_CMD_VLD 0xac
46#define SFLASHC_BURST_CFG 0xe0
47#define NAND_ERASED_CW_DETECT_CFG 0xe8
48#define NAND_ERASED_CW_DETECT_STATUS 0xec
49#define NAND_EBI2_ECC_BUF_CFG 0xf0
50#define FLASH_BUF_ACC 0x100
51
52#define NAND_CTRL 0xf00
53#define NAND_VERSION 0xf08
54#define NAND_READ_LOCATION_0 0xf20
55#define NAND_READ_LOCATION_1 0xf24
56
57/* dummy register offsets, used by write_reg_dma */
58#define NAND_DEV_CMD1_RESTORE 0xdead
59#define NAND_DEV_CMD_VLD_RESTORE 0xbeef
60
61/* NAND_FLASH_CMD bits */
62#define PAGE_ACC BIT(4)
63#define LAST_PAGE BIT(5)
64
65/* NAND_FLASH_CHIP_SELECT bits */
66#define NAND_DEV_SEL 0
67#define DM_EN BIT(2)
68
69/* NAND_FLASH_STATUS bits */
70#define FS_OP_ERR BIT(4)
71#define FS_READY_BSY_N BIT(5)
72#define FS_MPU_ERR BIT(8)
73#define FS_DEVICE_STS_ERR BIT(16)
74#define FS_DEVICE_WP BIT(23)
75
76/* NAND_BUFFER_STATUS bits */
77#define BS_UNCORRECTABLE_BIT BIT(8)
78#define BS_CORRECTABLE_ERR_MSK 0x1f
79
80/* NAND_DEVn_CFG0 bits */
81#define DISABLE_STATUS_AFTER_WRITE 4
82#define CW_PER_PAGE 6
83#define UD_SIZE_BYTES 9
84#define ECC_PARITY_SIZE_BYTES_RS 19
85#define SPARE_SIZE_BYTES 23
86#define NUM_ADDR_CYCLES 27
87#define STATUS_BFR_READ 30
88#define SET_RD_MODE_AFTER_STATUS 31
89
90/* NAND_DEVn_CFG0 bits */
91#define DEV0_CFG1_ECC_DISABLE 0
92#define WIDE_FLASH 1
93#define NAND_RECOVERY_CYCLES 2
94#define CS_ACTIVE_BSY 5
95#define BAD_BLOCK_BYTE_NUM 6
96#define BAD_BLOCK_IN_SPARE_AREA 16
97#define WR_RD_BSY_GAP 17
98#define ENABLE_BCH_ECC 27
99
100/* NAND_DEV0_ECC_CFG bits */
101#define ECC_CFG_ECC_DISABLE 0
102#define ECC_SW_RESET 1
103#define ECC_MODE 4
104#define ECC_PARITY_SIZE_BYTES_BCH 8
105#define ECC_NUM_DATA_BYTES 16
106#define ECC_FORCE_CLK_OPEN 30
107
108/* NAND_DEV_CMD1 bits */
109#define READ_ADDR 0
110
111/* NAND_DEV_CMD_VLD bits */
112#define READ_START_VLD 0
113
114/* NAND_EBI2_ECC_BUF_CFG bits */
115#define NUM_STEPS 0
116
117/* NAND_ERASED_CW_DETECT_CFG bits */
118#define ERASED_CW_ECC_MASK 1
119#define AUTO_DETECT_RES 0
120#define MASK_ECC (1 << ERASED_CW_ECC_MASK)
121#define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
122#define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
123#define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
124#define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
125
126/* NAND_ERASED_CW_DETECT_STATUS bits */
127#define PAGE_ALL_ERASED BIT(7)
128#define CODEWORD_ALL_ERASED BIT(6)
129#define PAGE_ERASED BIT(5)
130#define CODEWORD_ERASED BIT(4)
131#define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
132#define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
133
134/* Version Mask */
135#define NAND_VERSION_MAJOR_MASK 0xf0000000
136#define NAND_VERSION_MAJOR_SHIFT 28
137#define NAND_VERSION_MINOR_MASK 0x0fff0000
138#define NAND_VERSION_MINOR_SHIFT 16
139
140/* NAND OP_CMDs */
141#define PAGE_READ 0x2
142#define PAGE_READ_WITH_ECC 0x3
143#define PAGE_READ_WITH_ECC_SPARE 0x4
144#define PROGRAM_PAGE 0x6
145#define PAGE_PROGRAM_WITH_ECC 0x7
146#define PROGRAM_PAGE_SPARE 0x9
147#define BLOCK_ERASE 0xa
148#define FETCH_ID 0xb
149#define RESET_DEVICE 0xd
150
151/*
152 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
153 * the driver calls the chunks 'step' or 'codeword' interchangeably
154 */
155#define NANDC_STEP_SIZE 512
156
157/*
158 * the largest page size we support is 8K, this will have 16 steps/codewords
159 * of 512 bytes each
160 */
161#define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
162
163/* we read at most 3 registers per codeword scan */
164#define MAX_REG_RD (3 * MAX_NUM_STEPS)
165
166/* ECC modes supported by the controller */
167#define ECC_NONE BIT(0)
168#define ECC_RS_4BIT BIT(1)
169#define ECC_BCH_4BIT BIT(2)
170#define ECC_BCH_8BIT BIT(3)
171
172struct desc_info {
173 struct list_head node;
174
175 enum dma_data_direction dir;
176 struct scatterlist sgl;
177 struct dma_async_tx_descriptor *dma_desc;
178};
179
180/*
181 * holds the current register values that we want to write. acts as a contiguous
182 * chunk of memory which we use to write the controller registers through DMA.
183 */
184struct nandc_regs {
185 __le32 cmd;
186 __le32 addr0;
187 __le32 addr1;
188 __le32 chip_sel;
189 __le32 exec;
190
191 __le32 cfg0;
192 __le32 cfg1;
193 __le32 ecc_bch_cfg;
194
195 __le32 clrflashstatus;
196 __le32 clrreadstatus;
197
198 __le32 cmd1;
199 __le32 vld;
200
201 __le32 orig_cmd1;
202 __le32 orig_vld;
203
204 __le32 ecc_buf_cfg;
205};
206
207/*
208 * NAND controller data struct
209 *
210 * @controller: base controller structure
211 * @host_list: list containing all the chips attached to the
212 * controller
213 * @dev: parent device
214 * @base: MMIO base
215 * @base_dma: physical base address of controller registers
216 * @core_clk: controller clock
217 * @aon_clk: another controller clock
218 *
219 * @chan: dma channel
220 * @cmd_crci: ADM DMA CRCI for command flow control
221 * @data_crci: ADM DMA CRCI for data flow control
222 * @desc_list: DMA descriptor list (list of desc_infos)
223 *
224 * @data_buffer: our local DMA buffer for page read/writes,
225 * used when we can't use the buffer provided
226 * by upper layers directly
227 * @buf_size/count/start: markers for chip->read_buf/write_buf functions
228 * @reg_read_buf: local buffer for reading back registers via DMA
229 * @reg_read_pos: marker for data read in reg_read_buf
230 *
231 * @regs: a contiguous chunk of memory for DMA register
232 * writes. contains the register values to be
233 * written to controller
234 * @cmd1/vld: some fixed controller register values
235 * @ecc_modes: supported ECC modes by the current controller,
236 * initialized via DT match data
237 */
238struct qcom_nand_controller {
239 struct nand_hw_control controller;
240 struct list_head host_list;
241
242 struct device *dev;
243
244 void __iomem *base;
245 dma_addr_t base_dma;
246
247 struct clk *core_clk;
248 struct clk *aon_clk;
249
250 struct dma_chan *chan;
251 unsigned int cmd_crci;
252 unsigned int data_crci;
253 struct list_head desc_list;
254
255 u8 *data_buffer;
256 int buf_size;
257 int buf_count;
258 int buf_start;
259
260 __le32 *reg_read_buf;
261 int reg_read_pos;
262
263 struct nandc_regs *regs;
264
265 u32 cmd1, vld;
266 u32 ecc_modes;
267};
268
269/*
270 * NAND chip structure
271 *
272 * @chip: base NAND chip structure
273 * @node: list node to add itself to host_list in
274 * qcom_nand_controller
275 *
276 * @cs: chip select value for this chip
277 * @cw_size: the number of bytes in a single step/codeword
278 * of a page, consisting of all data, ecc, spare
279 * and reserved bytes
280 * @cw_data: the number of bytes within a codeword protected
281 * by ECC
282 * @use_ecc: request the controller to use ECC for the
283 * upcoming read/write
284 * @bch_enabled: flag to tell whether BCH ECC mode is used
285 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
286 * chip
287 * @status: value to be returned if NAND_CMD_STATUS command
288 * is executed
289 * @last_command: keeps track of last command on this chip. used
290 * for reading correct status
291 *
292 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
293 * ecc/non-ecc mode for the current nand flash
294 * device
295 */
296struct qcom_nand_host {
297 struct nand_chip chip;
298 struct list_head node;
299
300 int cs;
301 int cw_size;
302 int cw_data;
303 bool use_ecc;
304 bool bch_enabled;
305 int ecc_bytes_hw;
306 int spare_bytes;
307 int bbm_size;
308 u8 status;
309 int last_command;
310
311 u32 cfg0, cfg1;
312 u32 cfg0_raw, cfg1_raw;
313 u32 ecc_buf_cfg;
314 u32 ecc_bch_cfg;
315 u32 clrflashstatus;
316 u32 clrreadstatus;
317};
318
319static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
320{
321 return container_of(chip, struct qcom_nand_host, chip);
322}
323
324static inline struct qcom_nand_controller *
325get_qcom_nand_controller(struct nand_chip *chip)
326{
327 return container_of(chip->controller, struct qcom_nand_controller,
328 controller);
329}
330
331static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
332{
333 return ioread32(nandc->base + offset);
334}
335
336static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
337 u32 val)
338{
339 iowrite32(val, nandc->base + offset);
340}
341
342static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
343{
344 switch (offset) {
345 case NAND_FLASH_CMD:
346 return ®s->cmd;
347 case NAND_ADDR0:
348 return ®s->addr0;
349 case NAND_ADDR1:
350 return ®s->addr1;
351 case NAND_FLASH_CHIP_SELECT:
352 return ®s->chip_sel;
353 case NAND_EXEC_CMD:
354 return ®s->exec;
355 case NAND_FLASH_STATUS:
356 return ®s->clrflashstatus;
357 case NAND_DEV0_CFG0:
358 return ®s->cfg0;
359 case NAND_DEV0_CFG1:
360 return ®s->cfg1;
361 case NAND_DEV0_ECC_CFG:
362 return ®s->ecc_bch_cfg;
363 case NAND_READ_STATUS:
364 return ®s->clrreadstatus;
365 case NAND_DEV_CMD1:
366 return ®s->cmd1;
367 case NAND_DEV_CMD1_RESTORE:
368 return ®s->orig_cmd1;
369 case NAND_DEV_CMD_VLD:
370 return ®s->vld;
371 case NAND_DEV_CMD_VLD_RESTORE:
372 return ®s->orig_vld;
373 case NAND_EBI2_ECC_BUF_CFG:
374 return ®s->ecc_buf_cfg;
375 default:
376 return NULL;
377 }
378}
379
380static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
381 u32 val)
382{
383 struct nandc_regs *regs = nandc->regs;
384 __le32 *reg;
385
386 reg = offset_to_nandc_reg(regs, offset);
387
388 if (reg)
389 *reg = cpu_to_le32(val);
390}
391
392/* helper to configure address register values */
393static void set_address(struct qcom_nand_host *host, u16 column, int page)
394{
395 struct nand_chip *chip = &host->chip;
396 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
397
398 if (chip->options & NAND_BUSWIDTH_16)
399 column >>= 1;
400
401 nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
402 nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
403}
404
405/*
406 * update_rw_regs: set up read/write register values, these will be
407 * written to the NAND controller registers via DMA
408 *
409 * @num_cw: number of steps for the read/write operation
410 * @read: read or write operation
411 */
412static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
413{
414 struct nand_chip *chip = &host->chip;
415 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
416 u32 cmd, cfg0, cfg1, ecc_bch_cfg;
417
418 if (read) {
419 if (host->use_ecc)
420 cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
421 else
422 cmd = PAGE_READ | PAGE_ACC | LAST_PAGE;
423 } else {
424 cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
425 }
426
427 if (host->use_ecc) {
428 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
429 (num_cw - 1) << CW_PER_PAGE;
430
431 cfg1 = host->cfg1;
432 ecc_bch_cfg = host->ecc_bch_cfg;
433 } else {
434 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
435 (num_cw - 1) << CW_PER_PAGE;
436
437 cfg1 = host->cfg1_raw;
438 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
439 }
440
441 nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
442 nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
443 nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
444 nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
445 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
446 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
447 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
448 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
449}
450
451static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
452 int reg_off, const void *vaddr, int size,
453 bool flow_control)
454{
455 struct desc_info *desc;
456 struct dma_async_tx_descriptor *dma_desc;
457 struct scatterlist *sgl;
458 struct dma_slave_config slave_conf;
459 enum dma_transfer_direction dir_eng;
460 int ret;
461
462 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
463 if (!desc)
464 return -ENOMEM;
465
466 sgl = &desc->sgl;
467
468 sg_init_one(sgl, vaddr, size);
469
470 if (read) {
471 dir_eng = DMA_DEV_TO_MEM;
472 desc->dir = DMA_FROM_DEVICE;
473 } else {
474 dir_eng = DMA_MEM_TO_DEV;
475 desc->dir = DMA_TO_DEVICE;
476 }
477
478 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
479 if (ret == 0) {
480 ret = -ENOMEM;
481 goto err;
482 }
483
484 memset(&slave_conf, 0x00, sizeof(slave_conf));
485
486 slave_conf.device_fc = flow_control;
487 if (read) {
488 slave_conf.src_maxburst = 16;
489 slave_conf.src_addr = nandc->base_dma + reg_off;
490 slave_conf.slave_id = nandc->data_crci;
491 } else {
492 slave_conf.dst_maxburst = 16;
493 slave_conf.dst_addr = nandc->base_dma + reg_off;
494 slave_conf.slave_id = nandc->cmd_crci;
495 }
496
497 ret = dmaengine_slave_config(nandc->chan, &slave_conf);
498 if (ret) {
499 dev_err(nandc->dev, "failed to configure dma channel\n");
500 goto err;
501 }
502
503 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
504 if (!dma_desc) {
505 dev_err(nandc->dev, "failed to prepare desc\n");
506 ret = -EINVAL;
507 goto err;
508 }
509
510 desc->dma_desc = dma_desc;
511
512 list_add_tail(&desc->node, &nandc->desc_list);
513
514 return 0;
515err:
516 kfree(desc);
517
518 return ret;
519}
520
521/*
522 * read_reg_dma: prepares a descriptor to read a given number of
523 * contiguous registers to the reg_read_buf pointer
524 *
525 * @first: offset of the first register in the contiguous block
526 * @num_regs: number of registers to read
527 */
528static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
529 int num_regs)
530{
531 bool flow_control = false;
532 void *vaddr;
533 int size;
534
535 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
536 flow_control = true;
537
538 size = num_regs * sizeof(u32);
539 vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
540 nandc->reg_read_pos += num_regs;
541
542 return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
543}
544
545/*
546 * write_reg_dma: prepares a descriptor to write a given number of
547 * contiguous registers
548 *
549 * @first: offset of the first register in the contiguous block
550 * @num_regs: number of registers to write
551 */
552static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
553 int num_regs)
554{
555 bool flow_control = false;
556 struct nandc_regs *regs = nandc->regs;
557 void *vaddr;
558 int size;
559
560 vaddr = offset_to_nandc_reg(regs, first);
561
562 if (first == NAND_FLASH_CMD)
563 flow_control = true;
564
565 if (first == NAND_DEV_CMD1_RESTORE)
566 first = NAND_DEV_CMD1;
567
568 if (first == NAND_DEV_CMD_VLD_RESTORE)
569 first = NAND_DEV_CMD_VLD;
570
571 size = num_regs * sizeof(u32);
572
573 return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
574}
575
576/*
577 * read_data_dma: prepares a DMA descriptor to transfer data from the
578 * controller's internal buffer to the buffer 'vaddr'
579 *
580 * @reg_off: offset within the controller's data buffer
581 * @vaddr: virtual address of the buffer we want to write to
582 * @size: DMA transaction size in bytes
583 */
584static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
585 const u8 *vaddr, int size)
586{
587 return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
588}
589
590/*
591 * write_data_dma: prepares a DMA descriptor to transfer data from
592 * 'vaddr' to the controller's internal buffer
593 *
594 * @reg_off: offset within the controller's data buffer
595 * @vaddr: virtual address of the buffer we want to read from
596 * @size: DMA transaction size in bytes
597 */
598static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
599 const u8 *vaddr, int size)
600{
601 return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
602}
603
604/*
605 * helper to prepare dma descriptors to configure registers needed for reading a
606 * codeword/step in a page
607 */
608static void config_cw_read(struct qcom_nand_controller *nandc)
609{
610 write_reg_dma(nandc, NAND_FLASH_CMD, 3);
611 write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
612 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
613
614 write_reg_dma(nandc, NAND_EXEC_CMD, 1);
615
616 read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
617 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
618}
619
620/*
621 * helpers to prepare dma descriptors used to configure registers needed for
622 * writing a codeword/step in a page
623 */
624static void config_cw_write_pre(struct qcom_nand_controller *nandc)
625{
626 write_reg_dma(nandc, NAND_FLASH_CMD, 3);
627 write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
628 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
629}
630
631static void config_cw_write_post(struct qcom_nand_controller *nandc)
632{
633 write_reg_dma(nandc, NAND_EXEC_CMD, 1);
634
635 read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
636
637 write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
638 write_reg_dma(nandc, NAND_READ_STATUS, 1);
639}
640
641/*
642 * the following functions are used within chip->cmdfunc() to perform different
643 * NAND_CMD_* commands
644 */
645
646/* sets up descriptors for NAND_CMD_PARAM */
647static int nandc_param(struct qcom_nand_host *host)
648{
649 struct nand_chip *chip = &host->chip;
650 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
651
652 /*
653 * NAND_CMD_PARAM is called before we know much about the FLASH chip
654 * in use. we configure the controller to perform a raw read of 512
655 * bytes to read onfi params
656 */
657 nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE);
658 nandc_set_reg(nandc, NAND_ADDR0, 0);
659 nandc_set_reg(nandc, NAND_ADDR1, 0);
660 nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
661 | 512 << UD_SIZE_BYTES
662 | 5 << NUM_ADDR_CYCLES
663 | 0 << SPARE_SIZE_BYTES);
664 nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
665 | 0 << CS_ACTIVE_BSY
666 | 17 << BAD_BLOCK_BYTE_NUM
667 | 1 << BAD_BLOCK_IN_SPARE_AREA
668 | 2 << WR_RD_BSY_GAP
669 | 0 << WIDE_FLASH
670 | 1 << DEV0_CFG1_ECC_DISABLE);
671 nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
672
673 /* configure CMD1 and VLD for ONFI param probing */
674 nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
675 (nandc->vld & ~(1 << READ_START_VLD))
676 | 0 << READ_START_VLD);
677 nandc_set_reg(nandc, NAND_DEV_CMD1,
678 (nandc->cmd1 & ~(0xFF << READ_ADDR))
679 | NAND_CMD_PARAM << READ_ADDR);
680
681 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
682
683 nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
684 nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
685
686 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
687 write_reg_dma(nandc, NAND_DEV_CMD1, 1);
688
689 nandc->buf_count = 512;
690 memset(nandc->data_buffer, 0xff, nandc->buf_count);
691
692 config_cw_read(nandc);
693
694 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
695 nandc->buf_count);
696
697 /* restore CMD1 and VLD regs */
698 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
699 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);
700
701 return 0;
702}
703
704/* sets up descriptors for NAND_CMD_ERASE1 */
705static int erase_block(struct qcom_nand_host *host, int page_addr)
706{
707 struct nand_chip *chip = &host->chip;
708 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
709
710 nandc_set_reg(nandc, NAND_FLASH_CMD,
711 BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
712 nandc_set_reg(nandc, NAND_ADDR0, page_addr);
713 nandc_set_reg(nandc, NAND_ADDR1, 0);
714 nandc_set_reg(nandc, NAND_DEV0_CFG0,
715 host->cfg0_raw & ~(7 << CW_PER_PAGE));
716 nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
717 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
718 nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
719 nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
720
721 write_reg_dma(nandc, NAND_FLASH_CMD, 3);
722 write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
723 write_reg_dma(nandc, NAND_EXEC_CMD, 1);
724
725 read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
726
727 write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
728 write_reg_dma(nandc, NAND_READ_STATUS, 1);
729
730 return 0;
731}
732
733/* sets up descriptors for NAND_CMD_READID */
734static int read_id(struct qcom_nand_host *host, int column)
735{
736 struct nand_chip *chip = &host->chip;
737 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
738
739 if (column == -1)
740 return 0;
741
742 nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
743 nandc_set_reg(nandc, NAND_ADDR0, column);
744 nandc_set_reg(nandc, NAND_ADDR1, 0);
745 nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
746 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
747
748 write_reg_dma(nandc, NAND_FLASH_CMD, 4);
749 write_reg_dma(nandc, NAND_EXEC_CMD, 1);
750
751 read_reg_dma(nandc, NAND_READ_ID, 1);
752
753 return 0;
754}
755
756/* sets up descriptors for NAND_CMD_RESET */
757static int reset(struct qcom_nand_host *host)
758{
759 struct nand_chip *chip = &host->chip;
760 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
761
762 nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
763 nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
764
765 write_reg_dma(nandc, NAND_FLASH_CMD, 1);
766 write_reg_dma(nandc, NAND_EXEC_CMD, 1);
767
768 read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
769
770 return 0;
771}
772
773/* helpers to submit/free our list of dma descriptors */
774static int submit_descs(struct qcom_nand_controller *nandc)
775{
776 struct desc_info *desc;
777 dma_cookie_t cookie = 0;
778
779 list_for_each_entry(desc, &nandc->desc_list, node)
780 cookie = dmaengine_submit(desc->dma_desc);
781
782 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
783 return -ETIMEDOUT;
784
785 return 0;
786}
787
788static void free_descs(struct qcom_nand_controller *nandc)
789{
790 struct desc_info *desc, *n;
791
792 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
793 list_del(&desc->node);
794 dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
795 kfree(desc);
796 }
797}
798
799/* reset the register read buffer for next NAND operation */
800static void clear_read_regs(struct qcom_nand_controller *nandc)
801{
802 nandc->reg_read_pos = 0;
803 memset(nandc->reg_read_buf, 0,
804 MAX_REG_RD * sizeof(*nandc->reg_read_buf));
805}
806
807static void pre_command(struct qcom_nand_host *host, int command)
808{
809 struct nand_chip *chip = &host->chip;
810 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
811
812 nandc->buf_count = 0;
813 nandc->buf_start = 0;
814 host->use_ecc = false;
815 host->last_command = command;
816
817 clear_read_regs(nandc);
818}
819
820/*
821 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
822 * privately maintained status byte, this status byte can be read after
823 * NAND_CMD_STATUS is called
824 */
825static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
826{
827 struct nand_chip *chip = &host->chip;
828 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
829 struct nand_ecc_ctrl *ecc = &chip->ecc;
830 int num_cw;
831 int i;
832
833 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
834
835 for (i = 0; i < num_cw; i++) {
836 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
837
838 if (flash_status & FS_MPU_ERR)
839 host->status &= ~NAND_STATUS_WP;
840
841 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
842 (flash_status &
843 FS_DEVICE_STS_ERR)))
844 host->status |= NAND_STATUS_FAIL;
845 }
846}
847
848static void post_command(struct qcom_nand_host *host, int command)
849{
850 struct nand_chip *chip = &host->chip;
851 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
852
853 switch (command) {
854 case NAND_CMD_READID:
855 memcpy(nandc->data_buffer, nandc->reg_read_buf,
856 nandc->buf_count);
857 break;
858 case NAND_CMD_PAGEPROG:
859 case NAND_CMD_ERASE1:
860 parse_erase_write_errors(host, command);
861 break;
862 default:
863 break;
864 }
865}
866
867/*
868 * Implements chip->cmdfunc. It's only used for a limited set of commands.
869 * The rest of the commands wouldn't be called by upper layers. For example,
870 * NAND_CMD_READOOB would never be called because we have our own versions
871 * of read_oob ops for nand_ecc_ctrl.
872 */
873static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
874 int column, int page_addr)
875{
876 struct nand_chip *chip = mtd_to_nand(mtd);
877 struct qcom_nand_host *host = to_qcom_nand_host(chip);
878 struct nand_ecc_ctrl *ecc = &chip->ecc;
879 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
880 bool wait = false;
881 int ret = 0;
882
883 pre_command(host, command);
884
885 switch (command) {
886 case NAND_CMD_RESET:
887 ret = reset(host);
888 wait = true;
889 break;
890
891 case NAND_CMD_READID:
892 nandc->buf_count = 4;
893 ret = read_id(host, column);
894 wait = true;
895 break;
896
897 case NAND_CMD_PARAM:
898 ret = nandc_param(host);
899 wait = true;
900 break;
901
902 case NAND_CMD_ERASE1:
903 ret = erase_block(host, page_addr);
904 wait = true;
905 break;
906
907 case NAND_CMD_READ0:
908 /* we read the entire page for now */
909 WARN_ON(column != 0);
910
911 host->use_ecc = true;
912 set_address(host, 0, page_addr);
913 update_rw_regs(host, ecc->steps, true);
914 break;
915
916 case NAND_CMD_SEQIN:
917 WARN_ON(column != 0);
918 set_address(host, 0, page_addr);
919 break;
920
921 case NAND_CMD_PAGEPROG:
922 case NAND_CMD_STATUS:
923 case NAND_CMD_NONE:
924 default:
925 break;
926 }
927
928 if (ret) {
929 dev_err(nandc->dev, "failure executing command %d\n",
930 command);
931 free_descs(nandc);
932 return;
933 }
934
935 if (wait) {
936 ret = submit_descs(nandc);
937 if (ret)
938 dev_err(nandc->dev,
939 "failure submitting descs for command %d\n",
940 command);
941 }
942
943 free_descs(nandc);
944
945 post_command(host, command);
946}
947
948/*
949 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
950 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
951 *
952 * when using RS ECC, the HW reports the same erros when reading an erased CW,
953 * but it notifies that it is an erased CW by placing special characters at
954 * certain offsets in the buffer.
955 *
956 * verify if the page is erased or not, and fix up the page for RS ECC by
957 * replacing the special characters with 0xff.
958 */
959static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
960{
961 u8 empty1, empty2;
962
963 /*
964 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
965 * is erased by looking for 0x54s at offsets 3 and 175 from the
966 * beginning of each codeword
967 */
968
969 empty1 = data_buf[3];
970 empty2 = data_buf[175];
971
972 /*
973 * if the erased codework markers, if they exist override them with
974 * 0xffs
975 */
976 if ((empty1 == 0x54 && empty2 == 0xff) ||
977 (empty1 == 0xff && empty2 == 0x54)) {
978 data_buf[3] = 0xff;
979 data_buf[175] = 0xff;
980 }
981
982 /*
983 * check if the entire chunk contains 0xffs or not. if it doesn't, then
984 * restore the original values at the special offsets
985 */
986 if (memchr_inv(data_buf, 0xff, data_len)) {
987 data_buf[3] = empty1;
988 data_buf[175] = empty2;
989
990 return false;
991 }
992
993 return true;
994}
995
996struct read_stats {
997 __le32 flash;
998 __le32 buffer;
999 __le32 erased_cw;
1000};
1001
1002/*
1003 * reads back status registers set by the controller to notify page read
1004 * errors. this is equivalent to what 'ecc->correct()' would do.
1005 */
1006static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1007 u8 *oob_buf)
1008{
1009 struct nand_chip *chip = &host->chip;
1010 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1011 struct mtd_info *mtd = nand_to_mtd(chip);
1012 struct nand_ecc_ctrl *ecc = &chip->ecc;
1013 unsigned int max_bitflips = 0;
1014 struct read_stats *buf;
1015 int i;
1016
1017 buf = (struct read_stats *)nandc->reg_read_buf;
1018
1019 for (i = 0; i < ecc->steps; i++, buf++) {
1020 u32 flash, buffer, erased_cw;
1021 int data_len, oob_len;
1022
1023 if (i == (ecc->steps - 1)) {
1024 data_len = ecc->size - ((ecc->steps - 1) << 2);
1025 oob_len = ecc->steps << 2;
1026 } else {
1027 data_len = host->cw_data;
1028 oob_len = 0;
1029 }
1030
1031 flash = le32_to_cpu(buf->flash);
1032 buffer = le32_to_cpu(buf->buffer);
1033 erased_cw = le32_to_cpu(buf->erased_cw);
1034
1035 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1036 bool erased;
1037
1038 /* ignore erased codeword errors */
1039 if (host->bch_enabled) {
1040 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1041 true : false;
1042 } else {
1043 erased = erased_chunk_check_and_fixup(data_buf,
1044 data_len);
1045 }
1046
1047 if (erased) {
1048 data_buf += data_len;
1049 if (oob_buf)
1050 oob_buf += oob_len + ecc->bytes;
1051 continue;
1052 }
1053
1054 if (buffer & BS_UNCORRECTABLE_BIT) {
1055 int ret, ecclen, extraooblen;
1056 void *eccbuf;
1057
1058 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1059 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1060 extraooblen = oob_buf ? oob_len : 0;
1061
1062 /*
1063 * make sure it isn't an erased page reported
1064 * as not-erased by HW because of a few bitflips
1065 */
1066 ret = nand_check_erased_ecc_chunk(data_buf,
1067 data_len, eccbuf, ecclen, oob_buf,
1068 extraooblen, ecc->strength);
1069 if (ret < 0) {
1070 mtd->ecc_stats.failed++;
1071 } else {
1072 mtd->ecc_stats.corrected += ret;
1073 max_bitflips =
1074 max_t(unsigned int, max_bitflips, ret);
1075 }
1076 }
1077 } else {
1078 unsigned int stat;
1079
1080 stat = buffer & BS_CORRECTABLE_ERR_MSK;
1081 mtd->ecc_stats.corrected += stat;
1082 max_bitflips = max(max_bitflips, stat);
1083 }
1084
1085 data_buf += data_len;
1086 if (oob_buf)
1087 oob_buf += oob_len + ecc->bytes;
1088 }
1089
1090 return max_bitflips;
1091}
1092
1093/*
1094 * helper to perform the actual page read operation, used by ecc->read_page(),
1095 * ecc->read_oob()
1096 */
1097static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1098 u8 *oob_buf)
1099{
1100 struct nand_chip *chip = &host->chip;
1101 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1102 struct nand_ecc_ctrl *ecc = &chip->ecc;
1103 int i, ret;
1104
1105 /* queue cmd descs for each codeword */
1106 for (i = 0; i < ecc->steps; i++) {
1107 int data_size, oob_size;
1108
1109 if (i == (ecc->steps - 1)) {
1110 data_size = ecc->size - ((ecc->steps - 1) << 2);
1111 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1112 host->spare_bytes;
1113 } else {
1114 data_size = host->cw_data;
1115 oob_size = host->ecc_bytes_hw + host->spare_bytes;
1116 }
1117
1118 config_cw_read(nandc);
1119
1120 if (data_buf)
1121 read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1122 data_size);
1123
1124 /*
1125 * when ecc is enabled, the controller doesn't read the real
1126 * or dummy bad block markers in each chunk. To maintain a
1127 * consistent layout across RAW and ECC reads, we just
1128 * leave the real/dummy BBM offsets empty (i.e, filled with
1129 * 0xffs)
1130 */
1131 if (oob_buf) {
1132 int j;
1133
1134 for (j = 0; j < host->bbm_size; j++)
1135 *oob_buf++ = 0xff;
1136
1137 read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1138 oob_buf, oob_size);
1139 }
1140
1141 if (data_buf)
1142 data_buf += data_size;
1143 if (oob_buf)
1144 oob_buf += oob_size;
1145 }
1146
1147 ret = submit_descs(nandc);
1148 if (ret)
1149 dev_err(nandc->dev, "failure to read page/oob\n");
1150
1151 free_descs(nandc);
1152
1153 return ret;
1154}
1155
1156/*
1157 * a helper that copies the last step/codeword of a page (containing free oob)
1158 * into our local buffer
1159 */
1160static int copy_last_cw(struct qcom_nand_host *host, int page)
1161{
1162 struct nand_chip *chip = &host->chip;
1163 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1164 struct nand_ecc_ctrl *ecc = &chip->ecc;
1165 int size;
1166 int ret;
1167
1168 clear_read_regs(nandc);
1169
1170 size = host->use_ecc ? host->cw_data : host->cw_size;
1171
1172 /* prepare a clean read buffer */
1173 memset(nandc->data_buffer, 0xff, size);
1174
1175 set_address(host, host->cw_size * (ecc->steps - 1), page);
1176 update_rw_regs(host, 1, true);
1177
1178 config_cw_read(nandc);
1179
1180 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);
1181
1182 ret = submit_descs(nandc);
1183 if (ret)
1184 dev_err(nandc->dev, "failed to copy last codeword\n");
1185
1186 free_descs(nandc);
1187
1188 return ret;
1189}
1190
1191/* implements ecc->read_page() */
1192static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1193 uint8_t *buf, int oob_required, int page)
1194{
1195 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1196 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1197 u8 *data_buf, *oob_buf = NULL;
1198 int ret;
1199
1200 data_buf = buf;
1201 oob_buf = oob_required ? chip->oob_poi : NULL;
1202
1203 ret = read_page_ecc(host, data_buf, oob_buf);
1204 if (ret) {
1205 dev_err(nandc->dev, "failure to read page\n");
1206 return ret;
1207 }
1208
1209 return parse_read_errors(host, data_buf, oob_buf);
1210}
1211
1212/* implements ecc->read_page_raw() */
1213static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1214 struct nand_chip *chip, uint8_t *buf,
1215 int oob_required, int page)
1216{
1217 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1218 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1219 u8 *data_buf, *oob_buf;
1220 struct nand_ecc_ctrl *ecc = &chip->ecc;
1221 int i, ret;
1222
1223 data_buf = buf;
1224 oob_buf = chip->oob_poi;
1225
1226 host->use_ecc = false;
1227 update_rw_regs(host, ecc->steps, true);
1228
1229 for (i = 0; i < ecc->steps; i++) {
1230 int data_size1, data_size2, oob_size1, oob_size2;
1231 int reg_off = FLASH_BUF_ACC;
1232
1233 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1234 oob_size1 = host->bbm_size;
1235
1236 if (i == (ecc->steps - 1)) {
1237 data_size2 = ecc->size - data_size1 -
1238 ((ecc->steps - 1) << 2);
1239 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1240 host->spare_bytes;
1241 } else {
1242 data_size2 = host->cw_data - data_size1;
1243 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1244 }
1245
1246 config_cw_read(nandc);
1247
1248 read_data_dma(nandc, reg_off, data_buf, data_size1);
1249 reg_off += data_size1;
1250 data_buf += data_size1;
1251
1252 read_data_dma(nandc, reg_off, oob_buf, oob_size1);
1253 reg_off += oob_size1;
1254 oob_buf += oob_size1;
1255
1256 read_data_dma(nandc, reg_off, data_buf, data_size2);
1257 reg_off += data_size2;
1258 data_buf += data_size2;
1259
1260 read_data_dma(nandc, reg_off, oob_buf, oob_size2);
1261 oob_buf += oob_size2;
1262 }
1263
1264 ret = submit_descs(nandc);
1265 if (ret)
1266 dev_err(nandc->dev, "failure to read raw page\n");
1267
1268 free_descs(nandc);
1269
1270 return 0;
1271}
1272
1273/* implements ecc->read_oob() */
1274static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1275 int page)
1276{
1277 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1278 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1279 struct nand_ecc_ctrl *ecc = &chip->ecc;
1280 int ret;
1281
1282 clear_read_regs(nandc);
1283
1284 host->use_ecc = true;
1285 set_address(host, 0, page);
1286 update_rw_regs(host, ecc->steps, true);
1287
1288 ret = read_page_ecc(host, NULL, chip->oob_poi);
1289 if (ret)
1290 dev_err(nandc->dev, "failure to read oob\n");
1291
1292 return ret;
1293}
1294
1295/* implements ecc->write_page() */
1296static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1297 const uint8_t *buf, int oob_required, int page)
1298{
1299 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1300 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1301 struct nand_ecc_ctrl *ecc = &chip->ecc;
1302 u8 *data_buf, *oob_buf;
1303 int i, ret;
1304
1305 clear_read_regs(nandc);
1306
1307 data_buf = (u8 *)buf;
1308 oob_buf = chip->oob_poi;
1309
1310 host->use_ecc = true;
1311 update_rw_regs(host, ecc->steps, false);
1312
1313 for (i = 0; i < ecc->steps; i++) {
1314 int data_size, oob_size;
1315
1316 if (i == (ecc->steps - 1)) {
1317 data_size = ecc->size - ((ecc->steps - 1) << 2);
1318 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1319 host->spare_bytes;
1320 } else {
1321 data_size = host->cw_data;
1322 oob_size = ecc->bytes;
1323 }
1324
1325 config_cw_write_pre(nandc);
1326
1327 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);
1328
1329 /*
1330 * when ECC is enabled, we don't really need to write anything
1331 * to oob for the first n - 1 codewords since these oob regions
1332 * just contain ECC bytes that's written by the controller
1333 * itself. For the last codeword, we skip the bbm positions and
1334 * write to the free oob area.
1335 */
1336 if (i == (ecc->steps - 1)) {
1337 oob_buf += host->bbm_size;
1338
1339 write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1340 oob_buf, oob_size);
1341 }
1342
1343 config_cw_write_post(nandc);
1344
1345 data_buf += data_size;
1346 oob_buf += oob_size;
1347 }
1348
1349 ret = submit_descs(nandc);
1350 if (ret)
1351 dev_err(nandc->dev, "failure to write page\n");
1352
1353 free_descs(nandc);
1354
1355 return ret;
1356}
1357
1358/* implements ecc->write_page_raw() */
1359static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1360 struct nand_chip *chip, const uint8_t *buf,
1361 int oob_required, int page)
1362{
1363 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1364 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1365 struct nand_ecc_ctrl *ecc = &chip->ecc;
1366 u8 *data_buf, *oob_buf;
1367 int i, ret;
1368
1369 clear_read_regs(nandc);
1370
1371 data_buf = (u8 *)buf;
1372 oob_buf = chip->oob_poi;
1373
1374 host->use_ecc = false;
1375 update_rw_regs(host, ecc->steps, false);
1376
1377 for (i = 0; i < ecc->steps; i++) {
1378 int data_size1, data_size2, oob_size1, oob_size2;
1379 int reg_off = FLASH_BUF_ACC;
1380
1381 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1382 oob_size1 = host->bbm_size;
1383
1384 if (i == (ecc->steps - 1)) {
1385 data_size2 = ecc->size - data_size1 -
1386 ((ecc->steps - 1) << 2);
1387 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1388 host->spare_bytes;
1389 } else {
1390 data_size2 = host->cw_data - data_size1;
1391 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1392 }
1393
1394 config_cw_write_pre(nandc);
1395
1396 write_data_dma(nandc, reg_off, data_buf, data_size1);
1397 reg_off += data_size1;
1398 data_buf += data_size1;
1399
1400 write_data_dma(nandc, reg_off, oob_buf, oob_size1);
1401 reg_off += oob_size1;
1402 oob_buf += oob_size1;
1403
1404 write_data_dma(nandc, reg_off, data_buf, data_size2);
1405 reg_off += data_size2;
1406 data_buf += data_size2;
1407
1408 write_data_dma(nandc, reg_off, oob_buf, oob_size2);
1409 oob_buf += oob_size2;
1410
1411 config_cw_write_post(nandc);
1412 }
1413
1414 ret = submit_descs(nandc);
1415 if (ret)
1416 dev_err(nandc->dev, "failure to write raw page\n");
1417
1418 free_descs(nandc);
1419
1420 return ret;
1421}
1422
1423/*
1424 * implements ecc->write_oob()
1425 *
1426 * the NAND controller cannot write only data or only oob within a codeword,
1427 * since ecc is calculated for the combined codeword. we first copy the
1428 * entire contents for the last codeword(data + oob), replace the old oob
1429 * with the new one in chip->oob_poi, and then write the entire codeword.
1430 * this read-copy-write operation results in a slight performance loss.
1431 */
1432static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1433 int page)
1434{
1435 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1436 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1437 struct nand_ecc_ctrl *ecc = &chip->ecc;
1438 u8 *oob = chip->oob_poi;
1439 int data_size, oob_size;
1440 int ret, status = 0;
1441
1442 host->use_ecc = true;
1443
1444 ret = copy_last_cw(host, page);
1445 if (ret)
1446 return ret;
1447
1448 clear_read_regs(nandc);
1449
1450 /* calculate the data and oob size for the last codeword/step */
1451 data_size = ecc->size - ((ecc->steps - 1) << 2);
1452 oob_size = mtd->oobavail;
1453
1454 /* override new oob content to last codeword */
1455 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1456 0, mtd->oobavail);
1457
1458 set_address(host, host->cw_size * (ecc->steps - 1), page);
1459 update_rw_regs(host, 1, false);
1460
1461 config_cw_write_pre(nandc);
1462 write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1463 data_size + oob_size);
1464 config_cw_write_post(nandc);
1465
1466 ret = submit_descs(nandc);
1467
1468 free_descs(nandc);
1469
1470 if (ret) {
1471 dev_err(nandc->dev, "failure to write oob\n");
1472 return -EIO;
1473 }
1474
1475 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1476
1477 status = chip->waitfunc(mtd, chip);
1478
1479 return status & NAND_STATUS_FAIL ? -EIO : 0;
1480}
1481
1482static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1483{
1484 struct nand_chip *chip = mtd_to_nand(mtd);
1485 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1486 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1487 struct nand_ecc_ctrl *ecc = &chip->ecc;
1488 int page, ret, bbpos, bad = 0;
1489 u32 flash_status;
1490
1491 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1492
1493 /*
1494 * configure registers for a raw sub page read, the address is set to
1495 * the beginning of the last codeword, we don't care about reading ecc
1496 * portion of oob. we just want the first few bytes from this codeword
1497 * that contains the BBM
1498 */
1499 host->use_ecc = false;
1500
1501 ret = copy_last_cw(host, page);
1502 if (ret)
1503 goto err;
1504
1505 flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1506
1507 if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1508 dev_warn(nandc->dev, "error when trying to read BBM\n");
1509 goto err;
1510 }
1511
1512 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1513
1514 bad = nandc->data_buffer[bbpos] != 0xff;
1515
1516 if (chip->options & NAND_BUSWIDTH_16)
1517 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1518err:
1519 return bad;
1520}
1521
1522static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1523{
1524 struct nand_chip *chip = mtd_to_nand(mtd);
1525 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1526 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1527 struct nand_ecc_ctrl *ecc = &chip->ecc;
1528 int page, ret, status = 0;
1529
1530 clear_read_regs(nandc);
1531
1532 /*
1533 * to mark the BBM as bad, we flash the entire last codeword with 0s.
1534 * we don't care about the rest of the content in the codeword since
1535 * we aren't going to use this block again
1536 */
1537 memset(nandc->data_buffer, 0x00, host->cw_size);
1538
1539 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1540
1541 /* prepare write */
1542 host->use_ecc = false;
1543 set_address(host, host->cw_size * (ecc->steps - 1), page);
1544 update_rw_regs(host, 1, false);
1545
1546 config_cw_write_pre(nandc);
1547 write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
1548 config_cw_write_post(nandc);
1549
1550 ret = submit_descs(nandc);
1551
1552 free_descs(nandc);
1553
1554 if (ret) {
1555 dev_err(nandc->dev, "failure to update BBM\n");
1556 return -EIO;
1557 }
1558
1559 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1560
1561 status = chip->waitfunc(mtd, chip);
1562
1563 return status & NAND_STATUS_FAIL ? -EIO : 0;
1564}
1565
1566/*
1567 * the three functions below implement chip->read_byte(), chip->read_buf()
1568 * and chip->write_buf() respectively. these aren't used for
1569 * reading/writing page data, they are used for smaller data like reading
1570 * id, status etc
1571 */
1572static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
1573{
1574 struct nand_chip *chip = mtd_to_nand(mtd);
1575 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1576 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1577 u8 *buf = nandc->data_buffer;
1578 u8 ret = 0x0;
1579
1580 if (host->last_command == NAND_CMD_STATUS) {
1581 ret = host->status;
1582
1583 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
1584
1585 return ret;
1586 }
1587
1588 if (nandc->buf_start < nandc->buf_count)
1589 ret = buf[nandc->buf_start++];
1590
1591 return ret;
1592}
1593
1594static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
1595{
1596 struct nand_chip *chip = mtd_to_nand(mtd);
1597 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1598 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1599
1600 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
1601 nandc->buf_start += real_len;
1602}
1603
1604static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
1605 int len)
1606{
1607 struct nand_chip *chip = mtd_to_nand(mtd);
1608 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1609 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
1610
1611 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
1612
1613 nandc->buf_start += real_len;
1614}
1615
1616/* we support only one external chip for now */
1617static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
1618{
1619 struct nand_chip *chip = mtd_to_nand(mtd);
1620 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1621
1622 if (chipnr <= 0)
1623 return;
1624
1625 dev_warn(nandc->dev, "invalid chip select\n");
1626}
1627
1628/*
1629 * NAND controller page layout info
1630 *
1631 * Layout with ECC enabled:
1632 *
1633 * |----------------------| |---------------------------------|
1634 * | xx.......yy| | *********xx.......yy|
1635 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
1636 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
1637 * | xx.......yy| | *********xx.......yy|
1638 * |----------------------| |---------------------------------|
1639 * codeword 1,2..n-1 codeword n
1640 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
1641 *
1642 * n = Number of codewords in the page
1643 * . = ECC bytes
1644 * * = Spare/free bytes
1645 * x = Unused byte(s)
1646 * y = Reserved byte(s)
1647 *
1648 * 2K page: n = 4, spare = 16 bytes
1649 * 4K page: n = 8, spare = 32 bytes
1650 * 8K page: n = 16, spare = 64 bytes
1651 *
1652 * the qcom nand controller operates at a sub page/codeword level. each
1653 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
1654 * the number of ECC bytes vary based on the ECC strength and the bus width.
1655 *
1656 * the first n - 1 codewords contains 516 bytes of user data, the remaining
1657 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
1658 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
1659 *
1660 * When we access a page with ECC enabled, the reserved bytes(s) are not
1661 * accessible at all. When reading, we fill up these unreadable positions
1662 * with 0xffs. When writing, the controller skips writing the inaccessible
1663 * bytes.
1664 *
1665 * Layout with ECC disabled:
1666 *
1667 * |------------------------------| |---------------------------------------|
1668 * | yy xx.......| | bb *********xx.......|
1669 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
1670 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
1671 * | yy xx.......| | bb *********xx.......|
1672 * |------------------------------| |---------------------------------------|
1673 * codeword 1,2..n-1 codeword n
1674 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
1675 *
1676 * n = Number of codewords in the page
1677 * . = ECC bytes
1678 * * = Spare/free bytes
1679 * x = Unused byte(s)
1680 * y = Dummy Bad Bock byte(s)
1681 * b = Real Bad Block byte(s)
1682 * size1/size2 = function of codeword size and 'n'
1683 *
1684 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
1685 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
1686 * Block Markers. In the last codeword, this position contains the real BBM
1687 *
1688 * In order to have a consistent layout between RAW and ECC modes, we assume
1689 * the following OOB layout arrangement:
1690 *
1691 * |-----------| |--------------------|
1692 * |yyxx.......| |bb*********xx.......|
1693 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
1694 * |yyxx.......| |bb*********xx.......|
1695 * |yyxx.......| |bb*********xx.......|
1696 * |-----------| |--------------------|
1697 * first n - 1 nth OOB region
1698 * OOB regions
1699 *
1700 * n = Number of codewords in the page
1701 * . = ECC bytes
1702 * * = FREE OOB bytes
1703 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
1704 * x = Unused byte(s)
1705 * b = Real bad block byte(s) (inaccessible when ECC enabled)
1706 *
1707 * This layout is read as is when ECC is disabled. When ECC is enabled, the
1708 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
1709 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
1710 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
1711 * the sum of the three).
1712 */
1713static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1714 struct mtd_oob_region *oobregion)
1715{
1716 struct nand_chip *chip = mtd_to_nand(mtd);
1717 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1718 struct nand_ecc_ctrl *ecc = &chip->ecc;
1719
1720 if (section > 1)
1721 return -ERANGE;
1722
1723 if (!section) {
1724 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
1725 host->bbm_size;
1726 oobregion->offset = 0;
1727 } else {
1728 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
1729 oobregion->offset = mtd->oobsize - oobregion->length;
1730 }
1731
1732 return 0;
1733}
1734
1735static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
1736 struct mtd_oob_region *oobregion)
1737{
1738 struct nand_chip *chip = mtd_to_nand(mtd);
1739 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1740 struct nand_ecc_ctrl *ecc = &chip->ecc;
1741
1742 if (section)
1743 return -ERANGE;
1744
1745 oobregion->length = ecc->steps * 4;
1746 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
1747
1748 return 0;
1749}
1750
1751static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
1752 .ecc = qcom_nand_ooblayout_ecc,
1753 .free = qcom_nand_ooblayout_free,
1754};
1755
1756static int qcom_nand_host_setup(struct qcom_nand_host *host)
1757{
1758 struct nand_chip *chip = &host->chip;
1759 struct mtd_info *mtd = nand_to_mtd(chip);
1760 struct nand_ecc_ctrl *ecc = &chip->ecc;
1761 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1762 int cwperpage, bad_block_byte;
1763 bool wide_bus;
1764 int ecc_mode = 1;
1765
1766 /*
1767 * the controller requires each step consists of 512 bytes of data.
1768 * bail out if DT has populated a wrong step size.
1769 */
1770 if (ecc->size != NANDC_STEP_SIZE) {
1771 dev_err(nandc->dev, "invalid ecc size\n");
1772 return -EINVAL;
1773 }
1774
1775 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
1776
1777 if (ecc->strength >= 8) {
1778 /* 8 bit ECC defaults to BCH ECC on all platforms */
1779 host->bch_enabled = true;
1780 ecc_mode = 1;
1781
1782 if (wide_bus) {
1783 host->ecc_bytes_hw = 14;
1784 host->spare_bytes = 0;
1785 host->bbm_size = 2;
1786 } else {
1787 host->ecc_bytes_hw = 13;
1788 host->spare_bytes = 2;
1789 host->bbm_size = 1;
1790 }
1791 } else {
1792 /*
1793 * if the controller supports BCH for 4 bit ECC, the controller
1794 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
1795 * always 10 bytes
1796 */
1797 if (nandc->ecc_modes & ECC_BCH_4BIT) {
1798 /* BCH */
1799 host->bch_enabled = true;
1800 ecc_mode = 0;
1801
1802 if (wide_bus) {
1803 host->ecc_bytes_hw = 8;
1804 host->spare_bytes = 2;
1805 host->bbm_size = 2;
1806 } else {
1807 host->ecc_bytes_hw = 7;
1808 host->spare_bytes = 4;
1809 host->bbm_size = 1;
1810 }
1811 } else {
1812 /* RS */
1813 host->ecc_bytes_hw = 10;
1814
1815 if (wide_bus) {
1816 host->spare_bytes = 0;
1817 host->bbm_size = 2;
1818 } else {
1819 host->spare_bytes = 1;
1820 host->bbm_size = 1;
1821 }
1822 }
1823 }
1824
1825 /*
1826 * we consider ecc->bytes as the sum of all the non-data content in a
1827 * step. It gives us a clean representation of the oob area (even if
1828 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
1829 * ECC and 12 bytes for 4 bit ECC
1830 */
1831 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
1832
1833 ecc->read_page = qcom_nandc_read_page;
1834 ecc->read_page_raw = qcom_nandc_read_page_raw;
1835 ecc->read_oob = qcom_nandc_read_oob;
1836 ecc->write_page = qcom_nandc_write_page;
1837 ecc->write_page_raw = qcom_nandc_write_page_raw;
1838 ecc->write_oob = qcom_nandc_write_oob;
1839
1840 ecc->mode = NAND_ECC_HW;
1841
1842 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
1843
1844 cwperpage = mtd->writesize / ecc->size;
1845
1846 /*
1847 * DATA_UD_BYTES varies based on whether the read/write command protects
1848 * spare data with ECC too. We protect spare data by default, so we set
1849 * it to main + spare data, which are 512 and 4 bytes respectively.
1850 */
1851 host->cw_data = 516;
1852
1853 /*
1854 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
1855 * for 8 bit ECC
1856 */
1857 host->cw_size = host->cw_data + ecc->bytes;
1858
1859 if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
1860 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
1861 return -EINVAL;
1862 }
1863
1864 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
1865
1866 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
1867 | host->cw_data << UD_SIZE_BYTES
1868 | 0 << DISABLE_STATUS_AFTER_WRITE
1869 | 5 << NUM_ADDR_CYCLES
1870 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
1871 | 0 << STATUS_BFR_READ
1872 | 1 << SET_RD_MODE_AFTER_STATUS
1873 | host->spare_bytes << SPARE_SIZE_BYTES;
1874
1875 host->cfg1 = 7 << NAND_RECOVERY_CYCLES
1876 | 0 << CS_ACTIVE_BSY
1877 | bad_block_byte << BAD_BLOCK_BYTE_NUM
1878 | 0 << BAD_BLOCK_IN_SPARE_AREA
1879 | 2 << WR_RD_BSY_GAP
1880 | wide_bus << WIDE_FLASH
1881 | host->bch_enabled << ENABLE_BCH_ECC;
1882
1883 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
1884 | host->cw_size << UD_SIZE_BYTES
1885 | 5 << NUM_ADDR_CYCLES
1886 | 0 << SPARE_SIZE_BYTES;
1887
1888 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
1889 | 0 << CS_ACTIVE_BSY
1890 | 17 << BAD_BLOCK_BYTE_NUM
1891 | 1 << BAD_BLOCK_IN_SPARE_AREA
1892 | 2 << WR_RD_BSY_GAP
1893 | wide_bus << WIDE_FLASH
1894 | 1 << DEV0_CFG1_ECC_DISABLE;
1895
1896 host->ecc_bch_cfg = host->bch_enabled << ECC_CFG_ECC_DISABLE
1897 | 0 << ECC_SW_RESET
1898 | host->cw_data << ECC_NUM_DATA_BYTES
1899 | 1 << ECC_FORCE_CLK_OPEN
1900 | ecc_mode << ECC_MODE
1901 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
1902
1903 host->ecc_buf_cfg = 0x203 << NUM_STEPS;
1904
1905 host->clrflashstatus = FS_READY_BSY_N;
1906 host->clrreadstatus = 0xc0;
1907
1908 dev_dbg(nandc->dev,
1909 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
1910 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
1911 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
1912 cwperpage);
1913
1914 return 0;
1915}
1916
1917static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
1918{
1919 int ret;
1920
1921 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
1922 if (ret) {
1923 dev_err(nandc->dev, "failed to set DMA mask\n");
1924 return ret;
1925 }
1926
1927 /*
1928 * we use the internal buffer for reading ONFI params, reading small
1929 * data like ID and status, and preforming read-copy-write operations
1930 * when writing to a codeword partially. 532 is the maximum possible
1931 * size of a codeword for our nand controller
1932 */
1933 nandc->buf_size = 532;
1934
1935 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
1936 GFP_KERNEL);
1937 if (!nandc->data_buffer)
1938 return -ENOMEM;
1939
1940 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
1941 GFP_KERNEL);
1942 if (!nandc->regs)
1943 return -ENOMEM;
1944
1945 nandc->reg_read_buf = devm_kzalloc(nandc->dev,
1946 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
1947 GFP_KERNEL);
1948 if (!nandc->reg_read_buf)
1949 return -ENOMEM;
1950
1951 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
1952 if (!nandc->chan) {
1953 dev_err(nandc->dev, "failed to request slave channel\n");
1954 return -ENODEV;
1955 }
1956
1957 INIT_LIST_HEAD(&nandc->desc_list);
1958 INIT_LIST_HEAD(&nandc->host_list);
1959
1960 nand_hw_control_init(&nandc->controller);
1961
1962 return 0;
1963}
1964
1965static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
1966{
1967 dma_release_channel(nandc->chan);
1968}
1969
1970/* one time setup of a few nand controller registers */
1971static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
1972{
1973 /* kill onenand */
1974 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
1975
1976 /* enable ADM DMA */
1977 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
1978
1979 /* save the original values of these registers */
1980 nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
1981 nandc->vld = nandc_read(nandc, NAND_DEV_CMD_VLD);
1982
1983 return 0;
1984}
1985
1986static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
1987 struct qcom_nand_host *host,
1988 struct device_node *dn)
1989{
1990 struct nand_chip *chip = &host->chip;
1991 struct mtd_info *mtd = nand_to_mtd(chip);
1992 struct device *dev = nandc->dev;
1993 int ret;
1994
1995 ret = of_property_read_u32(dn, "reg", &host->cs);
1996 if (ret) {
1997 dev_err(dev, "can't get chip-select\n");
1998 return -ENXIO;
1999 }
2000
2001 nand_set_flash_node(chip, dn);
2002 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2003 mtd->owner = THIS_MODULE;
2004 mtd->dev.parent = dev;
2005
2006 chip->cmdfunc = qcom_nandc_command;
2007 chip->select_chip = qcom_nandc_select_chip;
2008 chip->read_byte = qcom_nandc_read_byte;
2009 chip->read_buf = qcom_nandc_read_buf;
2010 chip->write_buf = qcom_nandc_write_buf;
2011
2012 /*
2013 * the bad block marker is readable only when we read the last codeword
2014 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2015 * helpers don't allow us to read BB from a nand chip with ECC
2016 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2017 * and block_markbad helpers until we permanently switch to using
2018 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2019 */
2020 chip->block_bad = qcom_nandc_block_bad;
2021 chip->block_markbad = qcom_nandc_block_markbad;
2022
2023 chip->controller = &nandc->controller;
2024 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2025 NAND_SKIP_BBTSCAN;
2026
2027 /* set up initial status value */
2028 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2029
2030 ret = nand_scan_ident(mtd, 1, NULL);
2031 if (ret)
2032 return ret;
2033
2034 ret = qcom_nand_host_setup(host);
2035 if (ret)
2036 return ret;
2037
2038 ret = nand_scan_tail(mtd);
2039 if (ret)
2040 return ret;
2041
2042 return mtd_device_register(mtd, NULL, 0);
2043}
2044
2045/* parse custom DT properties here */
2046static int qcom_nandc_parse_dt(struct platform_device *pdev)
2047{
2048 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2049 struct device_node *np = nandc->dev->of_node;
2050 int ret;
2051
2052 ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
2053 if (ret) {
2054 dev_err(nandc->dev, "command CRCI unspecified\n");
2055 return ret;
2056 }
2057
2058 ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
2059 if (ret) {
2060 dev_err(nandc->dev, "data CRCI unspecified\n");
2061 return ret;
2062 }
2063
2064 return 0;
2065}
2066
2067static int qcom_nandc_probe(struct platform_device *pdev)
2068{
2069 struct qcom_nand_controller *nandc;
2070 struct qcom_nand_host *host;
2071 const void *dev_data;
2072 struct device *dev = &pdev->dev;
2073 struct device_node *dn = dev->of_node, *child;
2074 struct resource *res;
2075 int ret;
2076
2077 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2078 if (!nandc)
2079 return -ENOMEM;
2080
2081 platform_set_drvdata(pdev, nandc);
2082 nandc->dev = dev;
2083
2084 dev_data = of_device_get_match_data(dev);
2085 if (!dev_data) {
2086 dev_err(&pdev->dev, "failed to get device data\n");
2087 return -ENODEV;
2088 }
2089
2090 nandc->ecc_modes = (unsigned long)dev_data;
2091
2092 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2093 nandc->base = devm_ioremap_resource(dev, res);
2094 if (IS_ERR(nandc->base))
2095 return PTR_ERR(nandc->base);
2096
2097 nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2098
2099 nandc->core_clk = devm_clk_get(dev, "core");
2100 if (IS_ERR(nandc->core_clk))
2101 return PTR_ERR(nandc->core_clk);
2102
2103 nandc->aon_clk = devm_clk_get(dev, "aon");
2104 if (IS_ERR(nandc->aon_clk))
2105 return PTR_ERR(nandc->aon_clk);
2106
2107 ret = qcom_nandc_parse_dt(pdev);
2108 if (ret)
2109 return ret;
2110
2111 ret = qcom_nandc_alloc(nandc);
2112 if (ret)
2113 return ret;
2114
2115 ret = clk_prepare_enable(nandc->core_clk);
2116 if (ret)
2117 goto err_core_clk;
2118
2119 ret = clk_prepare_enable(nandc->aon_clk);
2120 if (ret)
2121 goto err_aon_clk;
2122
2123 ret = qcom_nandc_setup(nandc);
2124 if (ret)
2125 goto err_setup;
2126
2127 for_each_available_child_of_node(dn, child) {
2128 if (of_device_is_compatible(child, "qcom,nandcs")) {
2129 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2130 if (!host) {
2131 of_node_put(child);
2132 ret = -ENOMEM;
2133 goto err_cs_init;
2134 }
2135
2136 ret = qcom_nand_host_init(nandc, host, child);
2137 if (ret) {
2138 devm_kfree(dev, host);
2139 continue;
2140 }
2141
2142 list_add_tail(&host->node, &nandc->host_list);
2143 }
2144 }
2145
2146 if (list_empty(&nandc->host_list)) {
2147 ret = -ENODEV;
2148 goto err_cs_init;
2149 }
2150
2151 return 0;
2152
2153err_cs_init:
2154 list_for_each_entry(host, &nandc->host_list, node)
2155 nand_release(nand_to_mtd(&host->chip));
2156err_setup:
2157 clk_disable_unprepare(nandc->aon_clk);
2158err_aon_clk:
2159 clk_disable_unprepare(nandc->core_clk);
2160err_core_clk:
2161 qcom_nandc_unalloc(nandc);
2162
2163 return ret;
2164}
2165
2166static int qcom_nandc_remove(struct platform_device *pdev)
2167{
2168 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2169 struct qcom_nand_host *host;
2170
2171 list_for_each_entry(host, &nandc->host_list, node)
2172 nand_release(nand_to_mtd(&host->chip));
2173
2174 qcom_nandc_unalloc(nandc);
2175
2176 clk_disable_unprepare(nandc->aon_clk);
2177 clk_disable_unprepare(nandc->core_clk);
2178
2179 return 0;
2180}
2181
2182#define EBI2_NANDC_ECC_MODES (ECC_RS_4BIT | ECC_BCH_8BIT)
2183
2184/*
2185 * data will hold a struct pointer containing more differences once we support
2186 * more controller variants
2187 */
2188static const struct of_device_id qcom_nandc_of_match[] = {
2189 { .compatible = "qcom,ipq806x-nand",
2190 .data = (void *)EBI2_NANDC_ECC_MODES,
2191 },
2192 {}
2193};
2194MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2195
2196static struct platform_driver qcom_nandc_driver = {
2197 .driver = {
2198 .name = "qcom-nandc",
2199 .of_match_table = qcom_nandc_of_match,
2200 },
2201 .probe = qcom_nandc_probe,
2202 .remove = qcom_nandc_remove,
2203};
2204module_platform_driver(qcom_nandc_driver);
2205
2206MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2207MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2208MODULE_LICENSE("GPL v2");