Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Cadence NAND flash controller driver
4 *
5 * Copyright (C) 2019 Cadence
6 *
7 * Author: Piotr Sroka <piotrs@cadence.com>
8 */
9
10#include <linux/bitfield.h>
11#include <linux/clk.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h>
14#include <linux/interrupt.h>
15#include <linux/module.h>
16#include <linux/mtd/mtd.h>
17#include <linux/mtd/rawnand.h>
18#include <linux/iopoll.h>
19#include <linux/of.h>
20#include <linux/platform_device.h>
21#include <linux/property.h>
22#include <linux/slab.h>
23
24/*
25 * HPNFC can work in 3 modes:
26 * - PIO - can work in master or slave DMA
27 * - CDMA - needs Master DMA for accessing command descriptors.
28 * - Generic mode - can use only slave DMA.
29 * CDMA and PIO modes can be used to execute only base commands.
30 * Generic mode can be used to execute any command
31 * on NAND flash memory. Driver uses CDMA mode for
32 * block erasing, page reading, page programing.
33 * Generic mode is used for executing rest of commands.
34 */
35
36#define MAX_ADDRESS_CYC 6
37#define MAX_ERASE_ADDRESS_CYC 3
38#define MAX_DATA_SIZE 0xFFFC
39#define DMA_DATA_SIZE_ALIGN 8
40
41/* Register definition. */
42/*
43 * Command register 0.
44 * Writing data to this register will initiate a new transaction
45 * of the NF controller.
46 */
47#define CMD_REG0 0x0000
48/* Command type field mask. */
49#define CMD_REG0_CT GENMASK(31, 30)
50/* Command type CDMA. */
51#define CMD_REG0_CT_CDMA 0uL
52/* Command type generic. */
53#define CMD_REG0_CT_GEN 3uL
54/* Command thread number field mask. */
55#define CMD_REG0_TN GENMASK(27, 24)
56
57/* Command register 2. */
58#define CMD_REG2 0x0008
59/* Command register 3. */
60#define CMD_REG3 0x000C
61/* Pointer register to select which thread status will be selected. */
62#define CMD_STATUS_PTR 0x0010
63/* Command status register for selected thread. */
64#define CMD_STATUS 0x0014
65
66/* Interrupt status register. */
67#define INTR_STATUS 0x0110
68#define INTR_STATUS_SDMA_ERR BIT(22)
69#define INTR_STATUS_SDMA_TRIGG BIT(21)
70#define INTR_STATUS_UNSUPP_CMD BIT(19)
71#define INTR_STATUS_DDMA_TERR BIT(18)
72#define INTR_STATUS_CDMA_TERR BIT(17)
73#define INTR_STATUS_CDMA_IDL BIT(16)
74
75/* Interrupt enable register. */
76#define INTR_ENABLE 0x0114
77#define INTR_ENABLE_INTR_EN BIT(31)
78#define INTR_ENABLE_SDMA_ERR_EN BIT(22)
79#define INTR_ENABLE_SDMA_TRIGG_EN BIT(21)
80#define INTR_ENABLE_UNSUPP_CMD_EN BIT(19)
81#define INTR_ENABLE_DDMA_TERR_EN BIT(18)
82#define INTR_ENABLE_CDMA_TERR_EN BIT(17)
83#define INTR_ENABLE_CDMA_IDLE_EN BIT(16)
84
85/* Controller internal state. */
86#define CTRL_STATUS 0x0118
87#define CTRL_STATUS_INIT_COMP BIT(9)
88#define CTRL_STATUS_CTRL_BUSY BIT(8)
89
90/* Command Engine threads state. */
91#define TRD_STATUS 0x0120
92
93/* Command Engine interrupt thread error status. */
94#define TRD_ERR_INT_STATUS 0x0128
95/* Command Engine interrupt thread error enable. */
96#define TRD_ERR_INT_STATUS_EN 0x0130
97/* Command Engine interrupt thread complete status. */
98#define TRD_COMP_INT_STATUS 0x0138
99
100/*
101 * Transfer config 0 register.
102 * Configures data transfer parameters.
103 */
104#define TRAN_CFG_0 0x0400
105/* Offset value from the beginning of the page. */
106#define TRAN_CFG_0_OFFSET GENMASK(31, 16)
107/* Numbers of sectors to transfer within singlNF device's page. */
108#define TRAN_CFG_0_SEC_CNT GENMASK(7, 0)
109
110/*
111 * Transfer config 1 register.
112 * Configures data transfer parameters.
113 */
114#define TRAN_CFG_1 0x0404
115/* Size of last data sector. */
116#define TRAN_CFG_1_LAST_SEC_SIZE GENMASK(31, 16)
117/* Size of not-last data sector. */
118#define TRAN_CFG_1_SECTOR_SIZE GENMASK(15, 0)
119
120/* ECC engine configuration register 0. */
121#define ECC_CONFIG_0 0x0428
122/* Correction strength. */
123#define ECC_CONFIG_0_CORR_STR GENMASK(10, 8)
124/* Enable erased pages detection mechanism. */
125#define ECC_CONFIG_0_ERASE_DET_EN BIT(1)
126/* Enable controller ECC check bits generation and correction. */
127#define ECC_CONFIG_0_ECC_EN BIT(0)
128
129/* ECC engine configuration register 1. */
130#define ECC_CONFIG_1 0x042C
131
132/* Multiplane settings register. */
133#define MULTIPLANE_CFG 0x0434
134/* Cache operation settings. */
135#define CACHE_CFG 0x0438
136
137/* DMA settings register. */
138#define DMA_SETINGS 0x043C
139/* Enable SDMA error report on access unprepared slave DMA interface. */
140#define DMA_SETINGS_SDMA_ERR_RSP BIT(17)
141
142/* Transferred data block size for the slave DMA module. */
143#define SDMA_SIZE 0x0440
144
145/* Thread number associated with transferred data block
146 * for the slave DMA module.
147 */
148#define SDMA_TRD_NUM 0x0444
149/* Thread number mask. */
150#define SDMA_TRD_NUM_SDMA_TRD GENMASK(2, 0)
151
152#define CONTROL_DATA_CTRL 0x0494
153/* Thread number mask. */
154#define CONTROL_DATA_CTRL_SIZE GENMASK(15, 0)
155
156#define CTRL_VERSION 0x800
157#define CTRL_VERSION_REV GENMASK(7, 0)
158
159/* Available hardware features of the controller. */
160#define CTRL_FEATURES 0x804
161/* Support for NV-DDR2/3 work mode. */
162#define CTRL_FEATURES_NVDDR_2_3 BIT(28)
163/* Support for NV-DDR work mode. */
164#define CTRL_FEATURES_NVDDR BIT(27)
165/* Support for asynchronous work mode. */
166#define CTRL_FEATURES_ASYNC BIT(26)
167/* Support for asynchronous work mode. */
168#define CTRL_FEATURES_N_BANKS GENMASK(25, 24)
169/* Slave and Master DMA data width. */
170#define CTRL_FEATURES_DMA_DWITH64 BIT(21)
171/* Availability of Control Data feature.*/
172#define CTRL_FEATURES_CONTROL_DATA BIT(10)
173
174/* BCH Engine identification register 0 - correction strengths. */
175#define BCH_CFG_0 0x838
176#define BCH_CFG_0_CORR_CAP_0 GENMASK(7, 0)
177#define BCH_CFG_0_CORR_CAP_1 GENMASK(15, 8)
178#define BCH_CFG_0_CORR_CAP_2 GENMASK(23, 16)
179#define BCH_CFG_0_CORR_CAP_3 GENMASK(31, 24)
180
181/* BCH Engine identification register 1 - correction strengths. */
182#define BCH_CFG_1 0x83C
183#define BCH_CFG_1_CORR_CAP_4 GENMASK(7, 0)
184#define BCH_CFG_1_CORR_CAP_5 GENMASK(15, 8)
185#define BCH_CFG_1_CORR_CAP_6 GENMASK(23, 16)
186#define BCH_CFG_1_CORR_CAP_7 GENMASK(31, 24)
187
188/* BCH Engine identification register 2 - sector sizes. */
189#define BCH_CFG_2 0x840
190#define BCH_CFG_2_SECT_0 GENMASK(15, 0)
191#define BCH_CFG_2_SECT_1 GENMASK(31, 16)
192
193/* BCH Engine identification register 3. */
194#define BCH_CFG_3 0x844
195#define BCH_CFG_3_METADATA_SIZE GENMASK(23, 16)
196
197/* Ready/Busy# line status. */
198#define RBN_SETINGS 0x1004
199
200/* Common settings. */
201#define COMMON_SET 0x1008
202/* 16 bit device connected to the NAND Flash interface. */
203#define COMMON_SET_DEVICE_16BIT BIT(8)
204
205/* Skip_bytes registers. */
206#define SKIP_BYTES_CONF 0x100C
207#define SKIP_BYTES_MARKER_VALUE GENMASK(31, 16)
208#define SKIP_BYTES_NUM_OF_BYTES GENMASK(7, 0)
209
210#define SKIP_BYTES_OFFSET 0x1010
211#define SKIP_BYTES_OFFSET_VALUE GENMASK(23, 0)
212
213/* Timings configuration. */
214#define ASYNC_TOGGLE_TIMINGS 0x101c
215#define ASYNC_TOGGLE_TIMINGS_TRH GENMASK(28, 24)
216#define ASYNC_TOGGLE_TIMINGS_TRP GENMASK(20, 16)
217#define ASYNC_TOGGLE_TIMINGS_TWH GENMASK(12, 8)
218#define ASYNC_TOGGLE_TIMINGS_TWP GENMASK(4, 0)
219
220#define TIMINGS0 0x1024
221#define TIMINGS0_TADL GENMASK(31, 24)
222#define TIMINGS0_TCCS GENMASK(23, 16)
223#define TIMINGS0_TWHR GENMASK(15, 8)
224#define TIMINGS0_TRHW GENMASK(7, 0)
225
226#define TIMINGS1 0x1028
227#define TIMINGS1_TRHZ GENMASK(31, 24)
228#define TIMINGS1_TWB GENMASK(23, 16)
229#define TIMINGS1_TVDLY GENMASK(7, 0)
230
231#define TIMINGS2 0x102c
232#define TIMINGS2_TFEAT GENMASK(25, 16)
233#define TIMINGS2_CS_HOLD_TIME GENMASK(13, 8)
234#define TIMINGS2_CS_SETUP_TIME GENMASK(5, 0)
235
236/* Configuration of the resynchronization of slave DLL of PHY. */
237#define DLL_PHY_CTRL 0x1034
238#define DLL_PHY_CTRL_DLL_RST_N BIT(24)
239#define DLL_PHY_CTRL_EXTENDED_WR_MODE BIT(17)
240#define DLL_PHY_CTRL_EXTENDED_RD_MODE BIT(16)
241#define DLL_PHY_CTRL_RS_HIGH_WAIT_CNT GENMASK(11, 8)
242#define DLL_PHY_CTRL_RS_IDLE_CNT GENMASK(7, 0)
243
244/* Register controlling DQ related timing. */
245#define PHY_DQ_TIMING 0x2000
246/* Register controlling DSQ related timing. */
247#define PHY_DQS_TIMING 0x2004
248#define PHY_DQS_TIMING_DQS_SEL_OE_END GENMASK(3, 0)
249#define PHY_DQS_TIMING_PHONY_DQS_SEL BIT(16)
250#define PHY_DQS_TIMING_USE_PHONY_DQS BIT(20)
251
252/* Register controlling the gate and loopback control related timing. */
253#define PHY_GATE_LPBK_CTRL 0x2008
254#define PHY_GATE_LPBK_CTRL_RDS GENMASK(24, 19)
255
256/* Register holds the control for the master DLL logic. */
257#define PHY_DLL_MASTER_CTRL 0x200C
258#define PHY_DLL_MASTER_CTRL_BYPASS_MODE BIT(23)
259
260/* Register holds the control for the slave DLL logic. */
261#define PHY_DLL_SLAVE_CTRL 0x2010
262
263/* This register handles the global control settings for the PHY. */
264#define PHY_CTRL 0x2080
265#define PHY_CTRL_SDR_DQS BIT(14)
266#define PHY_CTRL_PHONY_DQS GENMASK(9, 4)
267
268/*
269 * This register handles the global control settings
270 * for the termination selects for reads.
271 */
272#define PHY_TSEL 0x2084
273
274/* Generic command layout. */
275#define GCMD_LAY_CS GENMASK_ULL(11, 8)
276/*
277 * This bit informs the minicotroller if it has to wait for tWB
278 * after sending the last CMD/ADDR/DATA in the sequence.
279 */
280#define GCMD_LAY_TWB BIT_ULL(6)
281/* Type of generic instruction. */
282#define GCMD_LAY_INSTR GENMASK_ULL(5, 0)
283
284/* Generic CMD sequence type. */
285#define GCMD_LAY_INSTR_CMD 0
286/* Generic ADDR sequence type. */
287#define GCMD_LAY_INSTR_ADDR 1
288/* Generic data transfer sequence type. */
289#define GCMD_LAY_INSTR_DATA 2
290
291/* Input part of generic command type of input is command. */
292#define GCMD_LAY_INPUT_CMD GENMASK_ULL(23, 16)
293
294/* Generic command address sequence - address fields. */
295#define GCMD_LAY_INPUT_ADDR GENMASK_ULL(63, 16)
296/* Generic command address sequence - address size. */
297#define GCMD_LAY_INPUT_ADDR_SIZE GENMASK_ULL(13, 11)
298
299/* Transfer direction field of generic command data sequence. */
300#define GCMD_DIR BIT_ULL(11)
301/* Read transfer direction of generic command data sequence. */
302#define GCMD_DIR_READ 0
303/* Write transfer direction of generic command data sequence. */
304#define GCMD_DIR_WRITE 1
305
306/* ECC enabled flag of generic command data sequence - ECC enabled. */
307#define GCMD_ECC_EN BIT_ULL(12)
308/* Generic command data sequence - sector size. */
309#define GCMD_SECT_SIZE GENMASK_ULL(31, 16)
310/* Generic command data sequence - sector count. */
311#define GCMD_SECT_CNT GENMASK_ULL(39, 32)
312/* Generic command data sequence - last sector size. */
313#define GCMD_LAST_SIZE GENMASK_ULL(55, 40)
314
315/* CDMA descriptor fields. */
316/* Erase command type of CDMA descriptor. */
317#define CDMA_CT_ERASE 0x1000
318/* Program page command type of CDMA descriptor. */
319#define CDMA_CT_WR 0x2100
320/* Read page command type of CDMA descriptor. */
321#define CDMA_CT_RD 0x2200
322
323/* Flash pointer memory shift. */
324#define CDMA_CFPTR_MEM_SHIFT 24
325/* Flash pointer memory mask. */
326#define CDMA_CFPTR_MEM GENMASK(26, 24)
327
328/*
329 * Command DMA descriptor flags. If set causes issue interrupt after
330 * the completion of descriptor processing.
331 */
332#define CDMA_CF_INT BIT(8)
333/*
334 * Command DMA descriptor flags - the next descriptor
335 * address field is valid and descriptor processing should continue.
336 */
337#define CDMA_CF_CONT BIT(9)
338/* DMA master flag of command DMA descriptor. */
339#define CDMA_CF_DMA_MASTER BIT(10)
340
341/* Operation complete status of command descriptor. */
342#define CDMA_CS_COMP BIT(15)
343/* Operation complete status of command descriptor. */
344/* Command descriptor status - operation fail. */
345#define CDMA_CS_FAIL BIT(14)
346/* Command descriptor status - page erased. */
347#define CDMA_CS_ERP BIT(11)
348/* Command descriptor status - timeout occurred. */
349#define CDMA_CS_TOUT BIT(10)
350/*
351 * Maximum amount of correction applied to one ECC sector.
352 * It is part of command descriptor status.
353 */
354#define CDMA_CS_MAXERR GENMASK(9, 2)
355/* Command descriptor status - uncorrectable ECC error. */
356#define CDMA_CS_UNCE BIT(1)
357/* Command descriptor status - descriptor error. */
358#define CDMA_CS_ERR BIT(0)
359
360/* Status of operation - OK. */
361#define STAT_OK 0
362/* Status of operation - FAIL. */
363#define STAT_FAIL 2
364/* Status of operation - uncorrectable ECC error. */
365#define STAT_ECC_UNCORR 3
366/* Status of operation - page erased. */
367#define STAT_ERASED 5
368/* Status of operation - correctable ECC error. */
369#define STAT_ECC_CORR 6
370/* Status of operation - unsuspected state. */
371#define STAT_UNKNOWN 7
372/* Status of operation - operation is not completed yet. */
373#define STAT_BUSY 0xFF
374
375#define BCH_MAX_NUM_CORR_CAPS 8
376#define BCH_MAX_NUM_SECTOR_SIZES 2
377
378struct cadence_nand_timings {
379 u32 async_toggle_timings;
380 u32 timings0;
381 u32 timings1;
382 u32 timings2;
383 u32 dll_phy_ctrl;
384 u32 phy_ctrl;
385 u32 phy_dqs_timing;
386 u32 phy_gate_lpbk_ctrl;
387};
388
389/* Command DMA descriptor. */
390struct cadence_nand_cdma_desc {
391 /* Next descriptor address. */
392 u64 next_pointer;
393
394 /* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
395 u32 flash_pointer;
396 /*field appears in HPNFC version 13*/
397 u16 bank;
398 u16 rsvd0;
399
400 /* Operation the controller needs to perform. */
401 u16 command_type;
402 u16 rsvd1;
403 /* Flags for operation of this command. */
404 u16 command_flags;
405 u16 rsvd2;
406
407 /* System/host memory address required for data DMA commands. */
408 u64 memory_pointer;
409
410 /* Status of operation. */
411 u32 status;
412 u32 rsvd3;
413
414 /* Address pointer to sync buffer location. */
415 u64 sync_flag_pointer;
416
417 /* Controls the buffer sync mechanism. */
418 u32 sync_arguments;
419 u32 rsvd4;
420
421 /* Control data pointer. */
422 u64 ctrl_data_ptr;
423};
424
425/* Interrupt status. */
426struct cadence_nand_irq_status {
427 /* Thread operation complete status. */
428 u32 trd_status;
429 /* Thread operation error. */
430 u32 trd_error;
431 /* Controller status. */
432 u32 status;
433};
434
435/* Cadence NAND flash controller capabilities get from driver data. */
436struct cadence_nand_dt_devdata {
437 /* Skew value of the output signals of the NAND Flash interface. */
438 u32 if_skew;
439 /* It informs if slave DMA interface is connected to DMA engine. */
440 unsigned int has_dma:1;
441};
442
443/* Cadence NAND flash controller capabilities read from registers. */
444struct cdns_nand_caps {
445 /* Maximum number of banks supported by hardware. */
446 u8 max_banks;
447 /* Slave and Master DMA data width in bytes (4 or 8). */
448 u8 data_dma_width;
449 /* Control Data feature supported. */
450 bool data_control_supp;
451 /* Is PHY type DLL. */
452 bool is_phy_type_dll;
453};
454
455struct cdns_nand_ctrl {
456 struct device *dev;
457 struct nand_controller controller;
458 struct cadence_nand_cdma_desc *cdma_desc;
459 /* IP capability. */
460 const struct cadence_nand_dt_devdata *caps1;
461 struct cdns_nand_caps caps2;
462 u8 ctrl_rev;
463 dma_addr_t dma_cdma_desc;
464 u8 *buf;
465 u32 buf_size;
466 u8 curr_corr_str_idx;
467
468 /* Register interface. */
469 void __iomem *reg;
470
471 struct {
472 void __iomem *virt;
473 dma_addr_t dma;
474 } io;
475
476 int irq;
477 /* Interrupts that have happened. */
478 struct cadence_nand_irq_status irq_status;
479 /* Interrupts we are waiting for. */
480 struct cadence_nand_irq_status irq_mask;
481 struct completion complete;
482 /* Protect irq_mask and irq_status. */
483 spinlock_t irq_lock;
484
485 int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
486 struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
487 struct nand_ecc_caps ecc_caps;
488
489 int curr_trans_type;
490
491 struct dma_chan *dmac;
492
493 u32 nf_clk_rate;
494 /*
495 * Estimated Board delay. The value includes the total
496 * round trip delay for the signals and is used for deciding on values
497 * associated with data read capture.
498 */
499 u32 board_delay;
500
501 struct nand_chip *selected_chip;
502
503 unsigned long assigned_cs;
504 struct list_head chips;
505 u8 bch_metadata_size;
506};
507
508struct cdns_nand_chip {
509 struct cadence_nand_timings timings;
510 struct nand_chip chip;
511 u8 nsels;
512 struct list_head node;
513
514 /*
515 * part of oob area of NAND flash memory page.
516 * This part is available for user to read or write.
517 */
518 u32 avail_oob_size;
519
520 /* Sector size. There are few sectors per mtd->writesize */
521 u32 sector_size;
522 u32 sector_count;
523
524 /* Offset of BBM. */
525 u8 bbm_offs;
526 /* Number of bytes reserved for BBM. */
527 u8 bbm_len;
528 /* ECC strength index. */
529 u8 corr_str_idx;
530
531 u8 cs[] __counted_by(nsels);
532};
533
534struct ecc_info {
535 int (*calc_ecc_bytes)(int step_size, int strength);
536 int max_step_size;
537};
538
539static inline struct
540cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
541{
542 return container_of(chip, struct cdns_nand_chip, chip);
543}
544
545static inline struct
546cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
547{
548 return container_of(controller, struct cdns_nand_ctrl, controller);
549}
550
551static bool
552cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
553 u32 buf_len)
554{
555 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
556
557 return buf && virt_addr_valid(buf) &&
558 likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
559 likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
560}
561
562static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
563 u32 reg_offset, u32 timeout_us,
564 u32 mask, bool is_clear)
565{
566 u32 val;
567 int ret;
568
569 ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
570 val, !(val & mask) == is_clear,
571 10, timeout_us);
572
573 if (ret < 0) {
574 dev_err(cdns_ctrl->dev,
575 "Timeout while waiting for reg %x with mask %x is clear %d\n",
576 reg_offset, mask, is_clear);
577 }
578
579 return ret;
580}
581
582static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
583 bool enable)
584{
585 u32 reg;
586
587 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
588 1000000,
589 CTRL_STATUS_CTRL_BUSY, true))
590 return -ETIMEDOUT;
591
592 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
593
594 if (enable)
595 reg |= ECC_CONFIG_0_ECC_EN;
596 else
597 reg &= ~ECC_CONFIG_0_ECC_EN;
598
599 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
600
601 return 0;
602}
603
604static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
605 u8 corr_str_idx)
606{
607 u32 reg;
608
609 if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
610 return;
611
612 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
613 reg &= ~ECC_CONFIG_0_CORR_STR;
614 reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
615 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
616
617 cdns_ctrl->curr_corr_str_idx = corr_str_idx;
618}
619
620static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
621 u8 strength)
622{
623 int i, corr_str_idx = -1;
624
625 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
626 if (cdns_ctrl->ecc_strengths[i] == strength) {
627 corr_str_idx = i;
628 break;
629 }
630 }
631
632 return corr_str_idx;
633}
634
635static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
636 u16 marker_value)
637{
638 u32 reg;
639
640 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
641 1000000,
642 CTRL_STATUS_CTRL_BUSY, true))
643 return -ETIMEDOUT;
644
645 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
646 reg &= ~SKIP_BYTES_MARKER_VALUE;
647 reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
648 marker_value);
649
650 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
651
652 return 0;
653}
654
655static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
656 u8 num_of_bytes,
657 u32 offset_value,
658 int enable)
659{
660 u32 reg, skip_bytes_offset;
661
662 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
663 1000000,
664 CTRL_STATUS_CTRL_BUSY, true))
665 return -ETIMEDOUT;
666
667 if (!enable) {
668 num_of_bytes = 0;
669 offset_value = 0;
670 }
671
672 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
673 reg &= ~SKIP_BYTES_NUM_OF_BYTES;
674 reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
675 num_of_bytes);
676 skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
677 offset_value);
678
679 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
680 writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
681
682 return 0;
683}
684
685/* Functions enables/disables hardware detection of erased data */
686static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
687 bool enable,
688 u8 bitflips_threshold)
689{
690 u32 reg;
691
692 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
693
694 if (enable)
695 reg |= ECC_CONFIG_0_ERASE_DET_EN;
696 else
697 reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
698
699 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
700 writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
701}
702
703static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
704 bool bit_bus16)
705{
706 u32 reg;
707
708 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
709 1000000,
710 CTRL_STATUS_CTRL_BUSY, true))
711 return -ETIMEDOUT;
712
713 reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
714
715 if (!bit_bus16)
716 reg &= ~COMMON_SET_DEVICE_16BIT;
717 else
718 reg |= COMMON_SET_DEVICE_16BIT;
719 writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
720
721 return 0;
722}
723
724static void
725cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
726 struct cadence_nand_irq_status *irq_status)
727{
728 writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
729 writel_relaxed(irq_status->trd_status,
730 cdns_ctrl->reg + TRD_COMP_INT_STATUS);
731 writel_relaxed(irq_status->trd_error,
732 cdns_ctrl->reg + TRD_ERR_INT_STATUS);
733}
734
735static void
736cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
737 struct cadence_nand_irq_status *irq_status)
738{
739 irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
740 irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
741 + TRD_COMP_INT_STATUS);
742 irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
743 + TRD_ERR_INT_STATUS);
744}
745
746static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
747 struct cadence_nand_irq_status *irq_status)
748{
749 cadence_nand_read_int_status(cdns_ctrl, irq_status);
750
751 return irq_status->status || irq_status->trd_status ||
752 irq_status->trd_error;
753}
754
755static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
756{
757 unsigned long flags;
758
759 spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
760 memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
761 memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
762 spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
763}
764
765/*
766 * This is the interrupt service routine. It handles all interrupts
767 * sent to this device.
768 */
769static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
770{
771 struct cdns_nand_ctrl *cdns_ctrl = dev_id;
772 struct cadence_nand_irq_status irq_status;
773 irqreturn_t result = IRQ_NONE;
774
775 spin_lock(&cdns_ctrl->irq_lock);
776
777 if (irq_detected(cdns_ctrl, &irq_status)) {
778 /* Handle interrupt. */
779 /* First acknowledge it. */
780 cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
781 /* Status in the device context for someone to read. */
782 cdns_ctrl->irq_status.status |= irq_status.status;
783 cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
784 cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
785 /* Notify anyone who cares that it happened. */
786 complete(&cdns_ctrl->complete);
787 /* Tell the OS that we've handled this. */
788 result = IRQ_HANDLED;
789 }
790 spin_unlock(&cdns_ctrl->irq_lock);
791
792 return result;
793}
794
795static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
796 struct cadence_nand_irq_status *irq_mask)
797{
798 writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
799 cdns_ctrl->reg + INTR_ENABLE);
800
801 writel_relaxed(irq_mask->trd_error,
802 cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
803}
804
805static void
806cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
807 struct cadence_nand_irq_status *irq_mask,
808 struct cadence_nand_irq_status *irq_status)
809{
810 unsigned long timeout = msecs_to_jiffies(10000);
811 unsigned long time_left;
812
813 time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
814 timeout);
815
816 *irq_status = cdns_ctrl->irq_status;
817 if (time_left == 0) {
818 /* Timeout error. */
819 dev_err(cdns_ctrl->dev, "timeout occurred:\n");
820 dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
821 irq_status->status, irq_mask->status);
822 dev_err(cdns_ctrl->dev,
823 "\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
824 irq_status->trd_status, irq_mask->trd_status);
825 dev_err(cdns_ctrl->dev,
826 "\t trd_error = 0x%x, trd_error mask = 0x%x\n",
827 irq_status->trd_error, irq_mask->trd_error);
828 }
829}
830
831/* Execute generic command on NAND controller. */
832static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
833 u8 chip_nr,
834 u64 mini_ctrl_cmd)
835{
836 u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
837
838 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
839 mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
840 mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
841
842 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
843 1000000,
844 CTRL_STATUS_CTRL_BUSY, true))
845 return -ETIMEDOUT;
846
847 cadence_nand_reset_irq(cdns_ctrl);
848
849 writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
850 writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
851
852 /* Select generic command. */
853 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
854 /* Thread number. */
855 reg |= FIELD_PREP(CMD_REG0_TN, 0);
856
857 /* Issue command. */
858 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
859
860 return 0;
861}
862
863/* Wait for data on slave DMA interface. */
864static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
865 u8 *out_sdma_trd,
866 u32 *out_sdma_size)
867{
868 struct cadence_nand_irq_status irq_mask, irq_status;
869
870 irq_mask.trd_status = 0;
871 irq_mask.trd_error = 0;
872 irq_mask.status = INTR_STATUS_SDMA_TRIGG
873 | INTR_STATUS_SDMA_ERR
874 | INTR_STATUS_UNSUPP_CMD;
875
876 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
877 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
878 if (irq_status.status == 0) {
879 dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
880 return -ETIMEDOUT;
881 }
882
883 if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
884 *out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
885 *out_sdma_trd = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
886 *out_sdma_trd =
887 FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
888 } else {
889 dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
890 irq_status.status);
891 return -EIO;
892 }
893
894 return 0;
895}
896
897static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
898{
899 u32 reg;
900
901 reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
902
903 cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
904
905 if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
906 cdns_ctrl->caps2.data_dma_width = 8;
907 else
908 cdns_ctrl->caps2.data_dma_width = 4;
909
910 if (reg & CTRL_FEATURES_CONTROL_DATA)
911 cdns_ctrl->caps2.data_control_supp = true;
912
913 if (reg & (CTRL_FEATURES_NVDDR_2_3
914 | CTRL_FEATURES_NVDDR))
915 cdns_ctrl->caps2.is_phy_type_dll = true;
916}
917
918/* Prepare CDMA descriptor. */
919static void
920cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
921 char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
922 dma_addr_t ctrl_data_ptr, u16 ctype)
923{
924 struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
925
926 memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
927
928 /* Set fields for one descriptor. */
929 cdma_desc->flash_pointer = flash_ptr;
930 if (cdns_ctrl->ctrl_rev >= 13)
931 cdma_desc->bank = nf_mem;
932 else
933 cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
934
935 cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
936 cdma_desc->command_flags |= CDMA_CF_INT;
937
938 cdma_desc->memory_pointer = mem_ptr;
939 cdma_desc->status = 0;
940 cdma_desc->sync_flag_pointer = 0;
941 cdma_desc->sync_arguments = 0;
942
943 cdma_desc->command_type = ctype;
944 cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
945}
946
947static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
948 u32 desc_status)
949{
950 if (desc_status & CDMA_CS_ERP)
951 return STAT_ERASED;
952
953 if (desc_status & CDMA_CS_UNCE)
954 return STAT_ECC_UNCORR;
955
956 if (desc_status & CDMA_CS_ERR) {
957 dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
958 return STAT_FAIL;
959 }
960
961 if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
962 return STAT_ECC_CORR;
963
964 return STAT_FAIL;
965}
966
967static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
968{
969 struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
970 u8 status = STAT_BUSY;
971
972 if (desc_ptr->status & CDMA_CS_FAIL) {
973 status = cadence_nand_check_desc_error(cdns_ctrl,
974 desc_ptr->status);
975 dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
976 } else if (desc_ptr->status & CDMA_CS_COMP) {
977 /* Descriptor finished with no errors. */
978 if (desc_ptr->command_flags & CDMA_CF_CONT) {
979 dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
980 status = STAT_UNKNOWN;
981 } else {
982 /* Last descriptor. */
983 status = STAT_OK;
984 }
985 }
986
987 return status;
988}
989
990static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
991 u8 thread)
992{
993 u32 reg;
994 int status;
995
996 /* Wait for thread ready. */
997 status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
998 1000000,
999 BIT(thread), true);
1000 if (status)
1001 return status;
1002
1003 cadence_nand_reset_irq(cdns_ctrl);
1004 reinit_completion(&cdns_ctrl->complete);
1005
1006 writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1007 cdns_ctrl->reg + CMD_REG2);
1008 writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1009
1010 /* Select CDMA mode. */
1011 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1012 /* Thread number. */
1013 reg |= FIELD_PREP(CMD_REG0_TN, thread);
1014 /* Issue command. */
1015 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1016
1017 return 0;
1018}
1019
1020/* Send SDMA command and wait for finish. */
1021static u32
1022cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1023 u8 thread)
1024{
1025 struct cadence_nand_irq_status irq_mask, irq_status = {0};
1026 int status;
1027
1028 irq_mask.trd_status = BIT(thread);
1029 irq_mask.trd_error = BIT(thread);
1030 irq_mask.status = INTR_STATUS_CDMA_TERR;
1031
1032 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1033
1034 status = cadence_nand_cdma_send(cdns_ctrl, thread);
1035 if (status)
1036 return status;
1037
1038 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1039
1040 if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1041 irq_status.trd_error == 0) {
1042 dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1043 return -ETIMEDOUT;
1044 }
1045 if (irq_status.status & irq_mask.status) {
1046 dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1047 return -EIO;
1048 }
1049
1050 return 0;
1051}
1052
1053/*
1054 * ECC size depends on configured ECC strength and on maximum supported
1055 * ECC step size.
1056 */
1057static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1058{
1059 int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1060
1061 return ALIGN(nbytes, 2);
1062}
1063
1064#define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1065 static int \
1066 cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1067 int strength)\
1068 {\
1069 return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1070 }
1071
1072CADENCE_NAND_CALC_ECC_BYTES(256)
1073CADENCE_NAND_CALC_ECC_BYTES(512)
1074CADENCE_NAND_CALC_ECC_BYTES(1024)
1075CADENCE_NAND_CALC_ECC_BYTES(2048)
1076CADENCE_NAND_CALC_ECC_BYTES(4096)
1077
1078/* Function reads BCH capabilities. */
1079static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1080{
1081 struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1082 int max_step_size = 0, nstrengths, i;
1083 u32 reg;
1084
1085 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1086 cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1087 if (cdns_ctrl->bch_metadata_size < 4) {
1088 dev_err(cdns_ctrl->dev,
1089 "Driver needs at least 4 bytes of BCH meta data\n");
1090 return -EIO;
1091 }
1092
1093 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1094 cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1095 cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1096 cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1097 cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1098
1099 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1100 cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1101 cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1102 cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1103 cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1104
1105 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1106 cdns_ctrl->ecc_stepinfos[0].stepsize =
1107 FIELD_GET(BCH_CFG_2_SECT_0, reg);
1108
1109 cdns_ctrl->ecc_stepinfos[1].stepsize =
1110 FIELD_GET(BCH_CFG_2_SECT_1, reg);
1111
1112 nstrengths = 0;
1113 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1114 if (cdns_ctrl->ecc_strengths[i] != 0)
1115 nstrengths++;
1116 }
1117
1118 ecc_caps->nstepinfos = 0;
1119 for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1120 /* ECC strengths are common for all step infos. */
1121 cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1122 cdns_ctrl->ecc_stepinfos[i].strengths =
1123 cdns_ctrl->ecc_strengths;
1124
1125 if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1126 ecc_caps->nstepinfos++;
1127
1128 if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1129 max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1130 }
1131 ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1132
1133 switch (max_step_size) {
1134 case 256:
1135 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1136 break;
1137 case 512:
1138 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1139 break;
1140 case 1024:
1141 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1142 break;
1143 case 2048:
1144 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1145 break;
1146 case 4096:
1147 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1148 break;
1149 default:
1150 dev_err(cdns_ctrl->dev,
1151 "Unsupported sector size(ecc step size) %d\n",
1152 max_step_size);
1153 return -EIO;
1154 }
1155
1156 return 0;
1157}
1158
1159/* Hardware initialization. */
1160static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1161{
1162 int status;
1163 u32 reg;
1164
1165 status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1166 1000000,
1167 CTRL_STATUS_INIT_COMP, false);
1168 if (status)
1169 return status;
1170
1171 reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1172 cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1173
1174 dev_info(cdns_ctrl->dev,
1175 "%s: cadence nand controller version reg %x\n",
1176 __func__, reg);
1177
1178 /* Disable cache and multiplane. */
1179 writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1180 writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1181
1182 /* Clear all interrupts. */
1183 writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1184
1185 cadence_nand_get_caps(cdns_ctrl);
1186 if (cadence_nand_read_bch_caps(cdns_ctrl))
1187 return -EIO;
1188
1189#ifndef CONFIG_64BIT
1190 if (cdns_ctrl->caps2.data_dma_width == 8) {
1191 dev_err(cdns_ctrl->dev,
1192 "cannot access 64-bit dma on !64-bit architectures");
1193 return -EIO;
1194 }
1195#endif
1196
1197 /*
1198 * Set IO width access to 8.
1199 * It is because during SW device discovering width access
1200 * is expected to be 8.
1201 */
1202 status = cadence_nand_set_access_width16(cdns_ctrl, false);
1203
1204 return status;
1205}
1206
1207#define TT_MAIN_OOB_AREAS 2
1208#define TT_RAW_PAGE 3
1209#define TT_BBM 4
1210#define TT_MAIN_OOB_AREA_EXT 5
1211
1212/* Prepare size of data to transfer. */
1213static void
1214cadence_nand_prepare_data_size(struct nand_chip *chip,
1215 int transfer_type)
1216{
1217 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1218 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1219 struct mtd_info *mtd = nand_to_mtd(chip);
1220 u32 sec_size = 0, offset = 0, sec_cnt = 1;
1221 u32 last_sec_size = cdns_chip->sector_size;
1222 u32 data_ctrl_size = 0;
1223 u32 reg = 0;
1224
1225 if (cdns_ctrl->curr_trans_type == transfer_type)
1226 return;
1227
1228 switch (transfer_type) {
1229 case TT_MAIN_OOB_AREA_EXT:
1230 sec_cnt = cdns_chip->sector_count;
1231 sec_size = cdns_chip->sector_size;
1232 data_ctrl_size = cdns_chip->avail_oob_size;
1233 break;
1234 case TT_MAIN_OOB_AREAS:
1235 sec_cnt = cdns_chip->sector_count;
1236 last_sec_size = cdns_chip->sector_size
1237 + cdns_chip->avail_oob_size;
1238 sec_size = cdns_chip->sector_size;
1239 break;
1240 case TT_RAW_PAGE:
1241 last_sec_size = mtd->writesize + mtd->oobsize;
1242 break;
1243 case TT_BBM:
1244 offset = mtd->writesize + cdns_chip->bbm_offs;
1245 last_sec_size = 8;
1246 break;
1247 }
1248
1249 reg = 0;
1250 reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1251 reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1252 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1253
1254 reg = 0;
1255 reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1256 reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1257 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1258
1259 if (cdns_ctrl->caps2.data_control_supp) {
1260 reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1261 reg &= ~CONTROL_DATA_CTRL_SIZE;
1262 reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1263 writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1264 }
1265
1266 cdns_ctrl->curr_trans_type = transfer_type;
1267}
1268
1269static int
1270cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1271 int page, void *buf, void *ctrl_dat, u32 buf_size,
1272 u32 ctrl_dat_size, enum dma_data_direction dir,
1273 bool with_ecc)
1274{
1275 dma_addr_t dma_buf, dma_ctrl_dat = 0;
1276 u8 thread_nr = chip_nr;
1277 int status;
1278 u16 ctype;
1279
1280 if (dir == DMA_FROM_DEVICE)
1281 ctype = CDMA_CT_RD;
1282 else
1283 ctype = CDMA_CT_WR;
1284
1285 cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1286
1287 dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1288 if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1289 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1290 return -EIO;
1291 }
1292
1293 if (ctrl_dat && ctrl_dat_size) {
1294 dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1295 ctrl_dat_size, dir);
1296 if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1297 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1298 buf_size, dir);
1299 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1300 return -EIO;
1301 }
1302 }
1303
1304 cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1305 dma_buf, dma_ctrl_dat, ctype);
1306
1307 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1308
1309 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1310 buf_size, dir);
1311
1312 if (ctrl_dat && ctrl_dat_size)
1313 dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1314 ctrl_dat_size, dir);
1315 if (status)
1316 return status;
1317
1318 return cadence_nand_cdma_finish(cdns_ctrl);
1319}
1320
1321static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1322 struct cadence_nand_timings *t)
1323{
1324 writel_relaxed(t->async_toggle_timings,
1325 cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1326 writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1327 writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1328 writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1329
1330 if (cdns_ctrl->caps2.is_phy_type_dll)
1331 writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1332
1333 writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1334
1335 if (cdns_ctrl->caps2.is_phy_type_dll) {
1336 writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1337 writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1338 writel_relaxed(t->phy_dqs_timing,
1339 cdns_ctrl->reg + PHY_DQS_TIMING);
1340 writel_relaxed(t->phy_gate_lpbk_ctrl,
1341 cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1342 writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1343 cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1344 writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1345 }
1346}
1347
1348static int cadence_nand_select_target(struct nand_chip *chip)
1349{
1350 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1351 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1352
1353 if (chip == cdns_ctrl->selected_chip)
1354 return 0;
1355
1356 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1357 1000000,
1358 CTRL_STATUS_CTRL_BUSY, true))
1359 return -ETIMEDOUT;
1360
1361 cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1362
1363 cadence_nand_set_ecc_strength(cdns_ctrl,
1364 cdns_chip->corr_str_idx);
1365
1366 cadence_nand_set_erase_detection(cdns_ctrl, true,
1367 chip->ecc.strength);
1368
1369 cdns_ctrl->curr_trans_type = -1;
1370 cdns_ctrl->selected_chip = chip;
1371
1372 return 0;
1373}
1374
1375static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1376{
1377 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1378 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1379 int status;
1380 u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1381
1382 cadence_nand_cdma_desc_prepare(cdns_ctrl,
1383 cdns_chip->cs[chip->cur_cs],
1384 page, 0, 0,
1385 CDMA_CT_ERASE);
1386 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1387 if (status) {
1388 dev_err(cdns_ctrl->dev, "erase operation failed\n");
1389 return -EIO;
1390 }
1391
1392 status = cadence_nand_cdma_finish(cdns_ctrl);
1393 if (status)
1394 return status;
1395
1396 return 0;
1397}
1398
1399static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1400{
1401 int status;
1402 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1403 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1404 struct mtd_info *mtd = nand_to_mtd(chip);
1405
1406 cadence_nand_prepare_data_size(chip, TT_BBM);
1407
1408 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1409
1410 /*
1411 * Read only bad block marker from offset
1412 * defined by a memory manufacturer.
1413 */
1414 status = cadence_nand_cdma_transfer(cdns_ctrl,
1415 cdns_chip->cs[chip->cur_cs],
1416 page, cdns_ctrl->buf, NULL,
1417 mtd->oobsize,
1418 0, DMA_FROM_DEVICE, false);
1419 if (status) {
1420 dev_err(cdns_ctrl->dev, "read BBM failed\n");
1421 return -EIO;
1422 }
1423
1424 memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1425
1426 return 0;
1427}
1428
1429static int cadence_nand_write_page(struct nand_chip *chip,
1430 const u8 *buf, int oob_required,
1431 int page)
1432{
1433 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1434 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1435 struct mtd_info *mtd = nand_to_mtd(chip);
1436 int status;
1437 u16 marker_val = 0xFFFF;
1438
1439 status = cadence_nand_select_target(chip);
1440 if (status)
1441 return status;
1442
1443 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1444 mtd->writesize
1445 + cdns_chip->bbm_offs,
1446 1);
1447
1448 if (oob_required) {
1449 marker_val = *(u16 *)(chip->oob_poi
1450 + cdns_chip->bbm_offs);
1451 } else {
1452 /* Set oob data to 0xFF. */
1453 memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1454 cdns_chip->avail_oob_size);
1455 }
1456
1457 cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1458
1459 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1460
1461 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1462 cdns_ctrl->caps2.data_control_supp) {
1463 u8 *oob;
1464
1465 if (oob_required)
1466 oob = chip->oob_poi;
1467 else
1468 oob = cdns_ctrl->buf + mtd->writesize;
1469
1470 status = cadence_nand_cdma_transfer(cdns_ctrl,
1471 cdns_chip->cs[chip->cur_cs],
1472 page, (void *)buf, oob,
1473 mtd->writesize,
1474 cdns_chip->avail_oob_size,
1475 DMA_TO_DEVICE, true);
1476 if (status) {
1477 dev_err(cdns_ctrl->dev, "write page failed\n");
1478 return -EIO;
1479 }
1480
1481 return 0;
1482 }
1483
1484 if (oob_required) {
1485 /* Transfer the data to the oob area. */
1486 memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1487 cdns_chip->avail_oob_size);
1488 }
1489
1490 memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1491
1492 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1493
1494 return cadence_nand_cdma_transfer(cdns_ctrl,
1495 cdns_chip->cs[chip->cur_cs],
1496 page, cdns_ctrl->buf, NULL,
1497 mtd->writesize
1498 + cdns_chip->avail_oob_size,
1499 0, DMA_TO_DEVICE, true);
1500}
1501
1502static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1503{
1504 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1505 struct mtd_info *mtd = nand_to_mtd(chip);
1506
1507 memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1508
1509 return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1510}
1511
1512static int cadence_nand_write_page_raw(struct nand_chip *chip,
1513 const u8 *buf, int oob_required,
1514 int page)
1515{
1516 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1517 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1518 struct mtd_info *mtd = nand_to_mtd(chip);
1519 int writesize = mtd->writesize;
1520 int oobsize = mtd->oobsize;
1521 int ecc_steps = chip->ecc.steps;
1522 int ecc_size = chip->ecc.size;
1523 int ecc_bytes = chip->ecc.bytes;
1524 void *tmp_buf = cdns_ctrl->buf;
1525 int oob_skip = cdns_chip->bbm_len;
1526 size_t size = writesize + oobsize;
1527 int i, pos, len;
1528 int status = 0;
1529
1530 status = cadence_nand_select_target(chip);
1531 if (status)
1532 return status;
1533
1534 /*
1535 * Fill the buffer with 0xff first except the full page transfer.
1536 * This simplifies the logic.
1537 */
1538 if (!buf || !oob_required)
1539 memset(tmp_buf, 0xff, size);
1540
1541 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1542
1543 /* Arrange the buffer for syndrome payload/ecc layout. */
1544 if (buf) {
1545 for (i = 0; i < ecc_steps; i++) {
1546 pos = i * (ecc_size + ecc_bytes);
1547 len = ecc_size;
1548
1549 if (pos >= writesize)
1550 pos += oob_skip;
1551 else if (pos + len > writesize)
1552 len = writesize - pos;
1553
1554 memcpy(tmp_buf + pos, buf, len);
1555 buf += len;
1556 if (len < ecc_size) {
1557 len = ecc_size - len;
1558 memcpy(tmp_buf + writesize + oob_skip, buf,
1559 len);
1560 buf += len;
1561 }
1562 }
1563 }
1564
1565 if (oob_required) {
1566 const u8 *oob = chip->oob_poi;
1567 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1568 (cdns_chip->sector_size + chip->ecc.bytes)
1569 + cdns_chip->sector_size + oob_skip;
1570
1571 /* BBM at the beginning of the OOB area. */
1572 memcpy(tmp_buf + writesize, oob, oob_skip);
1573
1574 /* OOB free. */
1575 memcpy(tmp_buf + oob_data_offset, oob,
1576 cdns_chip->avail_oob_size);
1577 oob += cdns_chip->avail_oob_size;
1578
1579 /* OOB ECC. */
1580 for (i = 0; i < ecc_steps; i++) {
1581 pos = ecc_size + i * (ecc_size + ecc_bytes);
1582 if (i == (ecc_steps - 1))
1583 pos += cdns_chip->avail_oob_size;
1584
1585 len = ecc_bytes;
1586
1587 if (pos >= writesize)
1588 pos += oob_skip;
1589 else if (pos + len > writesize)
1590 len = writesize - pos;
1591
1592 memcpy(tmp_buf + pos, oob, len);
1593 oob += len;
1594 if (len < ecc_bytes) {
1595 len = ecc_bytes - len;
1596 memcpy(tmp_buf + writesize + oob_skip, oob,
1597 len);
1598 oob += len;
1599 }
1600 }
1601 }
1602
1603 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1604
1605 return cadence_nand_cdma_transfer(cdns_ctrl,
1606 cdns_chip->cs[chip->cur_cs],
1607 page, cdns_ctrl->buf, NULL,
1608 mtd->writesize +
1609 mtd->oobsize,
1610 0, DMA_TO_DEVICE, false);
1611}
1612
1613static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1614 int page)
1615{
1616 return cadence_nand_write_page_raw(chip, NULL, true, page);
1617}
1618
1619static int cadence_nand_read_page(struct nand_chip *chip,
1620 u8 *buf, int oob_required, int page)
1621{
1622 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1623 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1624 struct mtd_info *mtd = nand_to_mtd(chip);
1625 int status = 0;
1626 int ecc_err_count = 0;
1627
1628 status = cadence_nand_select_target(chip);
1629 if (status)
1630 return status;
1631
1632 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1633 mtd->writesize
1634 + cdns_chip->bbm_offs, 1);
1635
1636 /*
1637 * If data buffer can be accessed by DMA and data_control feature
1638 * is supported then transfer data and oob directly.
1639 */
1640 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1641 cdns_ctrl->caps2.data_control_supp) {
1642 u8 *oob;
1643
1644 if (oob_required)
1645 oob = chip->oob_poi;
1646 else
1647 oob = cdns_ctrl->buf + mtd->writesize;
1648
1649 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1650 status = cadence_nand_cdma_transfer(cdns_ctrl,
1651 cdns_chip->cs[chip->cur_cs],
1652 page, buf, oob,
1653 mtd->writesize,
1654 cdns_chip->avail_oob_size,
1655 DMA_FROM_DEVICE, true);
1656 /* Otherwise use bounce buffer. */
1657 } else {
1658 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1659 status = cadence_nand_cdma_transfer(cdns_ctrl,
1660 cdns_chip->cs[chip->cur_cs],
1661 page, cdns_ctrl->buf,
1662 NULL, mtd->writesize
1663 + cdns_chip->avail_oob_size,
1664 0, DMA_FROM_DEVICE, true);
1665
1666 memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1667 if (oob_required)
1668 memcpy(chip->oob_poi,
1669 cdns_ctrl->buf + mtd->writesize,
1670 mtd->oobsize);
1671 }
1672
1673 switch (status) {
1674 case STAT_ECC_UNCORR:
1675 mtd->ecc_stats.failed++;
1676 ecc_err_count++;
1677 break;
1678 case STAT_ECC_CORR:
1679 ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1680 cdns_ctrl->cdma_desc->status);
1681 mtd->ecc_stats.corrected += ecc_err_count;
1682 break;
1683 case STAT_ERASED:
1684 case STAT_OK:
1685 break;
1686 default:
1687 dev_err(cdns_ctrl->dev, "read page failed\n");
1688 return -EIO;
1689 }
1690
1691 if (oob_required)
1692 if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1693 return -EIO;
1694
1695 return ecc_err_count;
1696}
1697
1698/* Reads OOB data from the device. */
1699static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1700{
1701 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1702
1703 return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1704}
1705
1706static int cadence_nand_read_page_raw(struct nand_chip *chip,
1707 u8 *buf, int oob_required, int page)
1708{
1709 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1710 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1711 struct mtd_info *mtd = nand_to_mtd(chip);
1712 int oob_skip = cdns_chip->bbm_len;
1713 int writesize = mtd->writesize;
1714 int ecc_steps = chip->ecc.steps;
1715 int ecc_size = chip->ecc.size;
1716 int ecc_bytes = chip->ecc.bytes;
1717 void *tmp_buf = cdns_ctrl->buf;
1718 int i, pos, len;
1719 int status = 0;
1720
1721 status = cadence_nand_select_target(chip);
1722 if (status)
1723 return status;
1724
1725 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1726
1727 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1728 status = cadence_nand_cdma_transfer(cdns_ctrl,
1729 cdns_chip->cs[chip->cur_cs],
1730 page, cdns_ctrl->buf, NULL,
1731 mtd->writesize
1732 + mtd->oobsize,
1733 0, DMA_FROM_DEVICE, false);
1734
1735 switch (status) {
1736 case STAT_ERASED:
1737 case STAT_OK:
1738 break;
1739 default:
1740 dev_err(cdns_ctrl->dev, "read raw page failed\n");
1741 return -EIO;
1742 }
1743
1744 /* Arrange the buffer for syndrome payload/ecc layout. */
1745 if (buf) {
1746 for (i = 0; i < ecc_steps; i++) {
1747 pos = i * (ecc_size + ecc_bytes);
1748 len = ecc_size;
1749
1750 if (pos >= writesize)
1751 pos += oob_skip;
1752 else if (pos + len > writesize)
1753 len = writesize - pos;
1754
1755 memcpy(buf, tmp_buf + pos, len);
1756 buf += len;
1757 if (len < ecc_size) {
1758 len = ecc_size - len;
1759 memcpy(buf, tmp_buf + writesize + oob_skip,
1760 len);
1761 buf += len;
1762 }
1763 }
1764 }
1765
1766 if (oob_required) {
1767 u8 *oob = chip->oob_poi;
1768 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1769 (cdns_chip->sector_size + chip->ecc.bytes)
1770 + cdns_chip->sector_size + oob_skip;
1771
1772 /* OOB free. */
1773 memcpy(oob, tmp_buf + oob_data_offset,
1774 cdns_chip->avail_oob_size);
1775
1776 /* BBM at the beginning of the OOB area. */
1777 memcpy(oob, tmp_buf + writesize, oob_skip);
1778
1779 oob += cdns_chip->avail_oob_size;
1780
1781 /* OOB ECC */
1782 for (i = 0; i < ecc_steps; i++) {
1783 pos = ecc_size + i * (ecc_size + ecc_bytes);
1784 len = ecc_bytes;
1785
1786 if (i == (ecc_steps - 1))
1787 pos += cdns_chip->avail_oob_size;
1788
1789 if (pos >= writesize)
1790 pos += oob_skip;
1791 else if (pos + len > writesize)
1792 len = writesize - pos;
1793
1794 memcpy(oob, tmp_buf + pos, len);
1795 oob += len;
1796 if (len < ecc_bytes) {
1797 len = ecc_bytes - len;
1798 memcpy(oob, tmp_buf + writesize + oob_skip,
1799 len);
1800 oob += len;
1801 }
1802 }
1803 }
1804
1805 return 0;
1806}
1807
1808static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1809 int page)
1810{
1811 return cadence_nand_read_page_raw(chip, NULL, true, page);
1812}
1813
1814static void cadence_nand_slave_dma_transfer_finished(void *data)
1815{
1816 struct completion *finished = data;
1817
1818 complete(finished);
1819}
1820
1821static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1822 void *buf,
1823 dma_addr_t dev_dma, size_t len,
1824 enum dma_data_direction dir)
1825{
1826 DECLARE_COMPLETION_ONSTACK(finished);
1827 struct dma_chan *chan;
1828 struct dma_device *dma_dev;
1829 dma_addr_t src_dma, dst_dma, buf_dma;
1830 struct dma_async_tx_descriptor *tx;
1831 dma_cookie_t cookie;
1832
1833 chan = cdns_ctrl->dmac;
1834 dma_dev = chan->device;
1835
1836 buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1837 if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1838 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1839 goto err;
1840 }
1841
1842 if (dir == DMA_FROM_DEVICE) {
1843 src_dma = cdns_ctrl->io.dma;
1844 dst_dma = buf_dma;
1845 } else {
1846 src_dma = buf_dma;
1847 dst_dma = cdns_ctrl->io.dma;
1848 }
1849
1850 tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1851 DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1852 if (!tx) {
1853 dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1854 goto err_unmap;
1855 }
1856
1857 tx->callback = cadence_nand_slave_dma_transfer_finished;
1858 tx->callback_param = &finished;
1859
1860 cookie = dmaengine_submit(tx);
1861 if (dma_submit_error(cookie)) {
1862 dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1863 goto err_unmap;
1864 }
1865
1866 dma_async_issue_pending(cdns_ctrl->dmac);
1867 wait_for_completion(&finished);
1868
1869 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1870
1871 return 0;
1872
1873err_unmap:
1874 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1875
1876err:
1877 dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1878
1879 return -EIO;
1880}
1881
1882static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1883 u8 *buf, int len)
1884{
1885 u8 thread_nr = 0;
1886 u32 sdma_size;
1887 int status;
1888
1889 /* Wait until slave DMA interface is ready to data transfer. */
1890 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1891 if (status)
1892 return status;
1893
1894 if (!cdns_ctrl->caps1->has_dma) {
1895 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1896
1897 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1898
1899 /* read alingment data */
1900 if (data_dma_width == 4)
1901 ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1902#ifdef CONFIG_64BIT
1903 else
1904 readsq(cdns_ctrl->io.virt, buf, len_in_words);
1905#endif
1906
1907 if (sdma_size > len) {
1908 int read_bytes = (data_dma_width == 4) ?
1909 len_in_words << 2 : len_in_words << 3;
1910
1911 /* read rest data from slave DMA interface if any */
1912 if (data_dma_width == 4)
1913 ioread32_rep(cdns_ctrl->io.virt,
1914 cdns_ctrl->buf,
1915 sdma_size / 4 - len_in_words);
1916#ifdef CONFIG_64BIT
1917 else
1918 readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1919 sdma_size / 8 - len_in_words);
1920#endif
1921
1922 /* copy rest of data */
1923 memcpy(buf + read_bytes, cdns_ctrl->buf,
1924 len - read_bytes);
1925 }
1926 return 0;
1927 }
1928
1929 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1930 status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1931 cdns_ctrl->io.dma,
1932 len, DMA_FROM_DEVICE);
1933 if (status == 0)
1934 return 0;
1935
1936 dev_warn(cdns_ctrl->dev,
1937 "Slave DMA transfer failed. Try again using bounce buffer.");
1938 }
1939
1940 /* If DMA transfer is not possible or failed then use bounce buffer. */
1941 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1942 cdns_ctrl->io.dma,
1943 sdma_size, DMA_FROM_DEVICE);
1944
1945 if (status) {
1946 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1947 return status;
1948 }
1949
1950 memcpy(buf, cdns_ctrl->buf, len);
1951
1952 return 0;
1953}
1954
1955static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1956 const u8 *buf, int len)
1957{
1958 u8 thread_nr = 0;
1959 u32 sdma_size;
1960 int status;
1961
1962 /* Wait until slave DMA interface is ready to data transfer. */
1963 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1964 if (status)
1965 return status;
1966
1967 if (!cdns_ctrl->caps1->has_dma) {
1968 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1969
1970 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1971
1972 if (data_dma_width == 4)
1973 iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1974#ifdef CONFIG_64BIT
1975 else
1976 writesq(cdns_ctrl->io.virt, buf, len_in_words);
1977#endif
1978
1979 if (sdma_size > len) {
1980 int written_bytes = (data_dma_width == 4) ?
1981 len_in_words << 2 : len_in_words << 3;
1982
1983 /* copy rest of data */
1984 memcpy(cdns_ctrl->buf, buf + written_bytes,
1985 len - written_bytes);
1986
1987 /* write all expected by nand controller data */
1988 if (data_dma_width == 4)
1989 iowrite32_rep(cdns_ctrl->io.virt,
1990 cdns_ctrl->buf,
1991 sdma_size / 4 - len_in_words);
1992#ifdef CONFIG_64BIT
1993 else
1994 writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1995 sdma_size / 8 - len_in_words);
1996#endif
1997 }
1998
1999 return 0;
2000 }
2001
2002 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
2003 status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
2004 cdns_ctrl->io.dma,
2005 len, DMA_TO_DEVICE);
2006 if (status == 0)
2007 return 0;
2008
2009 dev_warn(cdns_ctrl->dev,
2010 "Slave DMA transfer failed. Try again using bounce buffer.");
2011 }
2012
2013 /* If DMA transfer is not possible or failed then use bounce buffer. */
2014 memcpy(cdns_ctrl->buf, buf, len);
2015
2016 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
2017 cdns_ctrl->io.dma,
2018 sdma_size, DMA_TO_DEVICE);
2019
2020 if (status)
2021 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
2022
2023 return status;
2024}
2025
2026static int cadence_nand_force_byte_access(struct nand_chip *chip,
2027 bool force_8bit)
2028{
2029 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2030
2031 /*
2032 * Callers of this function do not verify if the NAND is using a 16-bit
2033 * an 8-bit bus for normal operations, so we need to take care of that
2034 * here by leaving the configuration unchanged if the NAND does not have
2035 * the NAND_BUSWIDTH_16 flag set.
2036 */
2037 if (!(chip->options & NAND_BUSWIDTH_16))
2038 return 0;
2039
2040 return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
2041}
2042
2043static int cadence_nand_cmd_opcode(struct nand_chip *chip,
2044 const struct nand_subop *subop)
2045{
2046 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2047 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2048 const struct nand_op_instr *instr;
2049 unsigned int op_id = 0;
2050 u64 mini_ctrl_cmd = 0;
2051 int ret;
2052
2053 instr = &subop->instrs[op_id];
2054
2055 if (instr->delay_ns > 0)
2056 mini_ctrl_cmd |= GCMD_LAY_TWB;
2057
2058 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2059 GCMD_LAY_INSTR_CMD);
2060 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2061 instr->ctx.cmd.opcode);
2062
2063 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2064 cdns_chip->cs[chip->cur_cs],
2065 mini_ctrl_cmd);
2066 if (ret)
2067 dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2068 instr->ctx.cmd.opcode);
2069
2070 return ret;
2071}
2072
2073static int cadence_nand_cmd_address(struct nand_chip *chip,
2074 const struct nand_subop *subop)
2075{
2076 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2077 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2078 const struct nand_op_instr *instr;
2079 unsigned int op_id = 0;
2080 u64 mini_ctrl_cmd = 0;
2081 unsigned int offset, naddrs;
2082 u64 address = 0;
2083 const u8 *addrs;
2084 int ret;
2085 int i;
2086
2087 instr = &subop->instrs[op_id];
2088
2089 if (instr->delay_ns > 0)
2090 mini_ctrl_cmd |= GCMD_LAY_TWB;
2091
2092 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2093 GCMD_LAY_INSTR_ADDR);
2094
2095 offset = nand_subop_get_addr_start_off(subop, op_id);
2096 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2097 addrs = &instr->ctx.addr.addrs[offset];
2098
2099 for (i = 0; i < naddrs; i++)
2100 address |= (u64)addrs[i] << (8 * i);
2101
2102 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2103 address);
2104 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2105 naddrs - 1);
2106
2107 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2108 cdns_chip->cs[chip->cur_cs],
2109 mini_ctrl_cmd);
2110 if (ret)
2111 dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2112
2113 return ret;
2114}
2115
2116static int cadence_nand_cmd_erase(struct nand_chip *chip,
2117 const struct nand_subop *subop)
2118{
2119 unsigned int op_id;
2120
2121 if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2122 int i;
2123 const struct nand_op_instr *instr = NULL;
2124 unsigned int offset, naddrs;
2125 const u8 *addrs;
2126 u32 page = 0;
2127
2128 instr = &subop->instrs[1];
2129 offset = nand_subop_get_addr_start_off(subop, 1);
2130 naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2131 addrs = &instr->ctx.addr.addrs[offset];
2132
2133 for (i = 0; i < naddrs; i++)
2134 page |= (u32)addrs[i] << (8 * i);
2135
2136 return cadence_nand_erase(chip, page);
2137 }
2138
2139 /*
2140 * If it is not an erase operation then handle operation
2141 * by calling exec_op function.
2142 */
2143 for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2144 int ret;
2145 const struct nand_operation nand_op = {
2146 .cs = chip->cur_cs,
2147 .instrs = &subop->instrs[op_id],
2148 .ninstrs = 1};
2149 ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2150 if (ret)
2151 return ret;
2152 }
2153
2154 return 0;
2155}
2156
2157static int cadence_nand_cmd_data(struct nand_chip *chip,
2158 const struct nand_subop *subop)
2159{
2160 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2161 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2162 const struct nand_op_instr *instr;
2163 unsigned int offset, op_id = 0;
2164 u64 mini_ctrl_cmd = 0;
2165 int len = 0;
2166 int ret;
2167
2168 instr = &subop->instrs[op_id];
2169
2170 if (instr->delay_ns > 0)
2171 mini_ctrl_cmd |= GCMD_LAY_TWB;
2172
2173 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2174 GCMD_LAY_INSTR_DATA);
2175
2176 if (instr->type == NAND_OP_DATA_OUT_INSTR)
2177 mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2178 GCMD_DIR_WRITE);
2179
2180 len = nand_subop_get_data_len(subop, op_id);
2181 offset = nand_subop_get_data_start_off(subop, op_id);
2182 mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2183 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2184 if (instr->ctx.data.force_8bit) {
2185 ret = cadence_nand_force_byte_access(chip, true);
2186 if (ret) {
2187 dev_err(cdns_ctrl->dev,
2188 "cannot change byte access generic data cmd failed\n");
2189 return ret;
2190 }
2191 }
2192
2193 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2194 cdns_chip->cs[chip->cur_cs],
2195 mini_ctrl_cmd);
2196 if (ret) {
2197 dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2198 return ret;
2199 }
2200
2201 if (instr->type == NAND_OP_DATA_IN_INSTR) {
2202 void *buf = instr->ctx.data.buf.in + offset;
2203
2204 ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2205 } else {
2206 const void *buf = instr->ctx.data.buf.out + offset;
2207
2208 ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2209 }
2210
2211 if (ret) {
2212 dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2213 return ret;
2214 }
2215
2216 if (instr->ctx.data.force_8bit) {
2217 ret = cadence_nand_force_byte_access(chip, false);
2218 if (ret) {
2219 dev_err(cdns_ctrl->dev,
2220 "cannot change byte access generic data cmd failed\n");
2221 }
2222 }
2223
2224 return ret;
2225}
2226
2227static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2228 const struct nand_subop *subop)
2229{
2230 int status;
2231 unsigned int op_id = 0;
2232 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2233 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2234 const struct nand_op_instr *instr = &subop->instrs[op_id];
2235 u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2236
2237 status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2238 timeout_us,
2239 BIT(cdns_chip->cs[chip->cur_cs]),
2240 false);
2241 return status;
2242}
2243
2244static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2245 NAND_OP_PARSER_PATTERN(
2246 cadence_nand_cmd_erase,
2247 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2248 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2249 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2250 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2251 NAND_OP_PARSER_PATTERN(
2252 cadence_nand_cmd_opcode,
2253 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2254 NAND_OP_PARSER_PATTERN(
2255 cadence_nand_cmd_address,
2256 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2257 NAND_OP_PARSER_PATTERN(
2258 cadence_nand_cmd_data,
2259 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2260 NAND_OP_PARSER_PATTERN(
2261 cadence_nand_cmd_data,
2262 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2263 NAND_OP_PARSER_PATTERN(
2264 cadence_nand_cmd_waitrdy,
2265 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2266 );
2267
2268static int cadence_nand_exec_op(struct nand_chip *chip,
2269 const struct nand_operation *op,
2270 bool check_only)
2271{
2272 if (!check_only) {
2273 int status = cadence_nand_select_target(chip);
2274
2275 if (status)
2276 return status;
2277 }
2278
2279 return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2280 check_only);
2281}
2282
2283static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2284 struct mtd_oob_region *oobregion)
2285{
2286 struct nand_chip *chip = mtd_to_nand(mtd);
2287 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2288
2289 if (section)
2290 return -ERANGE;
2291
2292 oobregion->offset = cdns_chip->bbm_len;
2293 oobregion->length = cdns_chip->avail_oob_size
2294 - cdns_chip->bbm_len;
2295
2296 return 0;
2297}
2298
2299static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2300 struct mtd_oob_region *oobregion)
2301{
2302 struct nand_chip *chip = mtd_to_nand(mtd);
2303 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2304
2305 if (section)
2306 return -ERANGE;
2307
2308 oobregion->offset = cdns_chip->avail_oob_size;
2309 oobregion->length = chip->ecc.total;
2310
2311 return 0;
2312}
2313
2314static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2315 .free = cadence_nand_ooblayout_free,
2316 .ecc = cadence_nand_ooblayout_ecc,
2317};
2318
2319static int calc_cycl(u32 timing, u32 clock)
2320{
2321 if (timing == 0 || clock == 0)
2322 return 0;
2323
2324 if ((timing % clock) > 0)
2325 return timing / clock;
2326 else
2327 return timing / clock - 1;
2328}
2329
2330/* Calculate max data valid window. */
2331static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2332 u32 board_delay_skew_min, u32 ext_mode)
2333{
2334 if (ext_mode == 0)
2335 clk_period /= 2;
2336
2337 return (trp_cnt + 1) * clk_period + trhoh_min +
2338 board_delay_skew_min;
2339}
2340
2341/* Calculate data valid window. */
2342static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2343 u32 trea_max, u32 ext_mode)
2344{
2345 if (ext_mode == 0)
2346 clk_period /= 2;
2347
2348 return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2349}
2350
2351static int
2352cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2353 const struct nand_interface_config *conf)
2354{
2355 const struct nand_sdr_timings *sdr;
2356 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2357 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2358 struct cadence_nand_timings *t = &cdns_chip->timings;
2359 u32 reg;
2360 u32 board_delay = cdns_ctrl->board_delay;
2361 u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2362 cdns_ctrl->nf_clk_rate);
2363 u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2364 u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2365 u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2366 u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2367 u32 if_skew = cdns_ctrl->caps1->if_skew;
2368 u32 board_delay_skew_min = board_delay - if_skew;
2369 u32 board_delay_skew_max = board_delay + if_skew;
2370 u32 dqs_sampl_res, phony_dqs_mod;
2371 u32 tdvw, tdvw_min, tdvw_max;
2372 u32 ext_rd_mode, ext_wr_mode;
2373 u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2374 u32 sampling_point;
2375
2376 sdr = nand_get_sdr_timings(conf);
2377 if (IS_ERR(sdr))
2378 return PTR_ERR(sdr);
2379
2380 memset(t, 0, sizeof(*t));
2381 /* Sampling point calculation. */
2382
2383 if (cdns_ctrl->caps2.is_phy_type_dll)
2384 phony_dqs_mod = 2;
2385 else
2386 phony_dqs_mod = 1;
2387
2388 dqs_sampl_res = clk_period / phony_dqs_mod;
2389
2390 tdvw_min = sdr->tREA_max + board_delay_skew_max;
2391 /*
2392 * The idea of those calculation is to get the optimum value
2393 * for tRP and tRH timings. If it is NOT possible to sample data
2394 * with optimal tRP/tRH settings, the parameters will be extended.
2395 * If clk_period is 50ns (the lowest value) this condition is met
2396 * for SDR timing modes 1, 2, 3, 4 and 5.
2397 * If clk_period is 20ns the condition is met only for SDR timing
2398 * mode 5.
2399 */
2400 if (sdr->tRC_min <= clk_period &&
2401 sdr->tRP_min <= (clk_period / 2) &&
2402 sdr->tREH_min <= (clk_period / 2)) {
2403 /* Performance mode. */
2404 ext_rd_mode = 0;
2405 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2406 sdr->tREA_max, ext_rd_mode);
2407 tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2408 board_delay_skew_min,
2409 ext_rd_mode);
2410 /*
2411 * Check if data valid window and sampling point can be found
2412 * and is not on the edge (ie. we have hold margin).
2413 * If not extend the tRP timings.
2414 */
2415 if (tdvw > 0) {
2416 if (tdvw_max <= tdvw_min ||
2417 (tdvw_max % dqs_sampl_res) == 0) {
2418 /*
2419 * No valid sampling point so the RE pulse need
2420 * to be widen widening by half clock cycle.
2421 */
2422 ext_rd_mode = 1;
2423 }
2424 } else {
2425 /*
2426 * There is no valid window
2427 * to be able to sample data the tRP need to be widen.
2428 * Very safe calculations are performed here.
2429 */
2430 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2431 + dqs_sampl_res) / clk_period;
2432 ext_rd_mode = 1;
2433 }
2434
2435 } else {
2436 /* Extended read mode. */
2437 u32 trh;
2438
2439 ext_rd_mode = 1;
2440 trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2441 trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2442 if (sdr->tREH_min >= trh)
2443 trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2444 else
2445 trh_cnt = calc_cycl(trh, clk_period);
2446
2447 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2448 sdr->tREA_max, ext_rd_mode);
2449 /*
2450 * Check if data valid window and sampling point can be found
2451 * or if it is at the edge check if previous is valid
2452 * - if not extend the tRP timings.
2453 */
2454 if (tdvw > 0) {
2455 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2456 sdr->tRHOH_min,
2457 board_delay_skew_min,
2458 ext_rd_mode);
2459
2460 if ((((tdvw_max / dqs_sampl_res)
2461 * dqs_sampl_res) <= tdvw_min) ||
2462 (((tdvw_max % dqs_sampl_res) == 0) &&
2463 (((tdvw_max / dqs_sampl_res - 1)
2464 * dqs_sampl_res) <= tdvw_min))) {
2465 /*
2466 * Data valid window width is lower than
2467 * sampling resolution and do not hit any
2468 * sampling point to be sure the sampling point
2469 * will be found the RE low pulse width will be
2470 * extended by one clock cycle.
2471 */
2472 trp_cnt = trp_cnt + 1;
2473 }
2474 } else {
2475 /*
2476 * There is no valid window to be able to sample data.
2477 * The tRP need to be widen.
2478 * Very safe calculations are performed here.
2479 */
2480 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2481 + dqs_sampl_res) / clk_period;
2482 }
2483 }
2484
2485 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2486 sdr->tRHOH_min,
2487 board_delay_skew_min, ext_rd_mode);
2488
2489 if (sdr->tWC_min <= clk_period &&
2490 (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2491 (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2492 ext_wr_mode = 0;
2493 } else {
2494 u32 twh;
2495
2496 ext_wr_mode = 1;
2497 twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2498 if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2499 twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2500 clk_period);
2501
2502 twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2503 if (sdr->tWH_min >= twh)
2504 twh = sdr->tWH_min;
2505
2506 twh_cnt = calc_cycl(twh + if_skew, clk_period);
2507 }
2508
2509 reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2510 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2511 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2512 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2513 t->async_toggle_timings = reg;
2514 dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2515
2516 tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2517 tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2518 twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2519 trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2520 reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2521
2522 /*
2523 * If timing exceeds delay field in timing register
2524 * then use maximum value.
2525 */
2526 if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2527 reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2528 else
2529 reg |= TIMINGS0_TCCS;
2530
2531 reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2532 reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2533 t->timings0 = reg;
2534 dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2535
2536 /* The following is related to single signal so skew is not needed. */
2537 trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2538 trhz_cnt = trhz_cnt + 1;
2539 twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2540 /*
2541 * Because of the two stage syncflop the value must be increased by 3
2542 * first value is related with sync, second value is related
2543 * with output if delay.
2544 */
2545 twb_cnt = twb_cnt + 3 + 5;
2546 /*
2547 * The following is related to the we edge of the random data input
2548 * sequence so skew is not needed.
2549 */
2550 tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2551 reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2552 reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2553 reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2554 t->timings1 = reg;
2555 dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2556
2557 tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2558 if (tfeat_cnt < twb_cnt)
2559 tfeat_cnt = twb_cnt;
2560
2561 tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2562 tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2563
2564 reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2565 reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2566 reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2567 t->timings2 = reg;
2568 dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2569
2570 if (cdns_ctrl->caps2.is_phy_type_dll) {
2571 reg = DLL_PHY_CTRL_DLL_RST_N;
2572 if (ext_wr_mode)
2573 reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2574 if (ext_rd_mode)
2575 reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2576
2577 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2578 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2579 t->dll_phy_ctrl = reg;
2580 dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2581 }
2582
2583 /* Sampling point calculation. */
2584 if ((tdvw_max % dqs_sampl_res) > 0)
2585 sampling_point = tdvw_max / dqs_sampl_res;
2586 else
2587 sampling_point = (tdvw_max / dqs_sampl_res - 1);
2588
2589 if (sampling_point * dqs_sampl_res > tdvw_min) {
2590 dll_phy_dqs_timing =
2591 FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2592 dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2593 phony_dqs_timing = sampling_point / phony_dqs_mod;
2594
2595 if ((sampling_point % 2) > 0) {
2596 dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2597 if ((tdvw_max % dqs_sampl_res) == 0)
2598 /*
2599 * Calculation for sampling point at the edge
2600 * of data and being odd number.
2601 */
2602 phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2603 / phony_dqs_mod - 1;
2604
2605 if (!cdns_ctrl->caps2.is_phy_type_dll)
2606 phony_dqs_timing--;
2607
2608 } else {
2609 phony_dqs_timing--;
2610 }
2611 rd_del_sel = phony_dqs_timing + 3;
2612 } else {
2613 dev_warn(cdns_ctrl->dev,
2614 "ERROR : cannot find valid sampling point\n");
2615 }
2616
2617 reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2618 if (cdns_ctrl->caps2.is_phy_type_dll)
2619 reg |= PHY_CTRL_SDR_DQS;
2620 t->phy_ctrl = reg;
2621 dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2622
2623 if (cdns_ctrl->caps2.is_phy_type_dll) {
2624 dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2625 dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2626 dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2627 dll_phy_dqs_timing);
2628 t->phy_dqs_timing = dll_phy_dqs_timing;
2629
2630 reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2631 dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2632 reg);
2633 t->phy_gate_lpbk_ctrl = reg;
2634
2635 dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2636 PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2637 dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2638 }
2639
2640 return 0;
2641}
2642
2643static int cadence_nand_attach_chip(struct nand_chip *chip)
2644{
2645 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2646 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2647 u32 ecc_size;
2648 struct mtd_info *mtd = nand_to_mtd(chip);
2649 int ret;
2650
2651 if (chip->options & NAND_BUSWIDTH_16) {
2652 ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2653 if (ret)
2654 return ret;
2655 }
2656
2657 chip->bbt_options |= NAND_BBT_USE_FLASH;
2658 chip->bbt_options |= NAND_BBT_NO_OOB;
2659 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2660
2661 chip->options |= NAND_NO_SUBPAGE_WRITE;
2662
2663 cdns_chip->bbm_offs = chip->badblockpos;
2664 cdns_chip->bbm_offs &= ~0x01;
2665 /* this value should be even number */
2666 cdns_chip->bbm_len = 2;
2667
2668 ret = nand_ecc_choose_conf(chip,
2669 &cdns_ctrl->ecc_caps,
2670 mtd->oobsize - cdns_chip->bbm_len);
2671 if (ret) {
2672 dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2673 return ret;
2674 }
2675
2676 dev_dbg(cdns_ctrl->dev,
2677 "chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2678 chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2679
2680 /* Error correction configuration. */
2681 cdns_chip->sector_size = chip->ecc.size;
2682 cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2683 ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2684
2685 cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2686
2687 if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2688 cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2689
2690 if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2691 > mtd->oobsize)
2692 cdns_chip->avail_oob_size -= 4;
2693
2694 ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2695 if (ret < 0)
2696 return -EINVAL;
2697
2698 cdns_chip->corr_str_idx = (u8)ret;
2699
2700 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2701 1000000,
2702 CTRL_STATUS_CTRL_BUSY, true))
2703 return -ETIMEDOUT;
2704
2705 cadence_nand_set_ecc_strength(cdns_ctrl,
2706 cdns_chip->corr_str_idx);
2707
2708 cadence_nand_set_erase_detection(cdns_ctrl, true,
2709 chip->ecc.strength);
2710
2711 /* Override the default read operations. */
2712 chip->ecc.read_page = cadence_nand_read_page;
2713 chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2714 chip->ecc.write_page = cadence_nand_write_page;
2715 chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2716 chip->ecc.read_oob = cadence_nand_read_oob;
2717 chip->ecc.write_oob = cadence_nand_write_oob;
2718 chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2719 chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2720
2721 if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2722 cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2723
2724 /* Is 32-bit DMA supported? */
2725 ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2726 if (ret) {
2727 dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2728 return ret;
2729 }
2730
2731 mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2732
2733 return 0;
2734}
2735
2736static const struct nand_controller_ops cadence_nand_controller_ops = {
2737 .attach_chip = cadence_nand_attach_chip,
2738 .exec_op = cadence_nand_exec_op,
2739 .setup_interface = cadence_nand_setup_interface,
2740};
2741
2742static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2743 struct device_node *np)
2744{
2745 struct cdns_nand_chip *cdns_chip;
2746 struct mtd_info *mtd;
2747 struct nand_chip *chip;
2748 int nsels, ret, i;
2749 u32 cs;
2750
2751 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2752 if (nsels <= 0) {
2753 dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2754 return -EINVAL;
2755 }
2756
2757 /* Allocate the nand chip structure. */
2758 cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2759 (nsels * sizeof(u8)),
2760 GFP_KERNEL);
2761 if (!cdns_chip) {
2762 dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2763 return -ENOMEM;
2764 }
2765
2766 cdns_chip->nsels = nsels;
2767
2768 for (i = 0; i < nsels; i++) {
2769 /* Retrieve CS id. */
2770 ret = of_property_read_u32_index(np, "reg", i, &cs);
2771 if (ret) {
2772 dev_err(cdns_ctrl->dev,
2773 "could not retrieve reg property: %d\n",
2774 ret);
2775 return ret;
2776 }
2777
2778 if (cs >= cdns_ctrl->caps2.max_banks) {
2779 dev_err(cdns_ctrl->dev,
2780 "invalid reg value: %u (max CS = %d)\n",
2781 cs, cdns_ctrl->caps2.max_banks);
2782 return -EINVAL;
2783 }
2784
2785 if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2786 dev_err(cdns_ctrl->dev,
2787 "CS %d already assigned\n", cs);
2788 return -EINVAL;
2789 }
2790
2791 cdns_chip->cs[i] = cs;
2792 }
2793
2794 chip = &cdns_chip->chip;
2795 chip->controller = &cdns_ctrl->controller;
2796 nand_set_flash_node(chip, np);
2797
2798 mtd = nand_to_mtd(chip);
2799 mtd->dev.parent = cdns_ctrl->dev;
2800
2801 /*
2802 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2803 * in the DT node, this entry will be overwritten in nand_scan_ident().
2804 */
2805 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2806
2807 ret = nand_scan(chip, cdns_chip->nsels);
2808 if (ret) {
2809 dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2810 return ret;
2811 }
2812
2813 ret = mtd_device_register(mtd, NULL, 0);
2814 if (ret) {
2815 dev_err(cdns_ctrl->dev,
2816 "failed to register mtd device: %d\n", ret);
2817 nand_cleanup(chip);
2818 return ret;
2819 }
2820
2821 list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2822
2823 return 0;
2824}
2825
2826static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2827{
2828 struct cdns_nand_chip *entry, *temp;
2829 struct nand_chip *chip;
2830 int ret;
2831
2832 list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2833 chip = &entry->chip;
2834 ret = mtd_device_unregister(nand_to_mtd(chip));
2835 WARN_ON(ret);
2836 nand_cleanup(chip);
2837 list_del(&entry->node);
2838 }
2839}
2840
2841static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2842{
2843 struct device_node *np = cdns_ctrl->dev->of_node;
2844 struct device_node *nand_np;
2845 int max_cs = cdns_ctrl->caps2.max_banks;
2846 int nchips, ret;
2847
2848 nchips = of_get_child_count(np);
2849
2850 if (nchips > max_cs) {
2851 dev_err(cdns_ctrl->dev,
2852 "too many NAND chips: %d (max = %d CS)\n",
2853 nchips, max_cs);
2854 return -EINVAL;
2855 }
2856
2857 for_each_child_of_node(np, nand_np) {
2858 ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2859 if (ret) {
2860 of_node_put(nand_np);
2861 cadence_nand_chips_cleanup(cdns_ctrl);
2862 return ret;
2863 }
2864 }
2865
2866 return 0;
2867}
2868
2869static void
2870cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2871{
2872 /* Disable interrupts. */
2873 writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2874}
2875
2876static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2877{
2878 dma_cap_mask_t mask;
2879 int ret;
2880
2881 cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2882 sizeof(*cdns_ctrl->cdma_desc),
2883 &cdns_ctrl->dma_cdma_desc,
2884 GFP_KERNEL);
2885 if (!cdns_ctrl->dma_cdma_desc)
2886 return -ENOMEM;
2887
2888 cdns_ctrl->buf_size = SZ_16K;
2889 cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2890 if (!cdns_ctrl->buf) {
2891 ret = -ENOMEM;
2892 goto free_buf_desc;
2893 }
2894
2895 if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2896 IRQF_SHARED, "cadence-nand-controller",
2897 cdns_ctrl)) {
2898 dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2899 ret = -ENODEV;
2900 goto free_buf;
2901 }
2902
2903 spin_lock_init(&cdns_ctrl->irq_lock);
2904 init_completion(&cdns_ctrl->complete);
2905
2906 ret = cadence_nand_hw_init(cdns_ctrl);
2907 if (ret)
2908 goto disable_irq;
2909
2910 dma_cap_zero(mask);
2911 dma_cap_set(DMA_MEMCPY, mask);
2912
2913 if (cdns_ctrl->caps1->has_dma) {
2914 cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);
2915 if (!cdns_ctrl->dmac) {
2916 dev_err(cdns_ctrl->dev,
2917 "Unable to get a DMA channel\n");
2918 ret = -EBUSY;
2919 goto disable_irq;
2920 }
2921 }
2922
2923 nand_controller_init(&cdns_ctrl->controller);
2924 INIT_LIST_HEAD(&cdns_ctrl->chips);
2925
2926 cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2927 cdns_ctrl->curr_corr_str_idx = 0xFF;
2928
2929 ret = cadence_nand_chips_init(cdns_ctrl);
2930 if (ret) {
2931 dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2932 ret);
2933 goto dma_release_chnl;
2934 }
2935
2936 kfree(cdns_ctrl->buf);
2937 cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2938 if (!cdns_ctrl->buf) {
2939 ret = -ENOMEM;
2940 goto dma_release_chnl;
2941 }
2942
2943 return 0;
2944
2945dma_release_chnl:
2946 if (cdns_ctrl->dmac)
2947 dma_release_channel(cdns_ctrl->dmac);
2948
2949disable_irq:
2950 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2951
2952free_buf:
2953 kfree(cdns_ctrl->buf);
2954
2955free_buf_desc:
2956 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2957 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2958
2959 return ret;
2960}
2961
2962/* Driver exit point. */
2963static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2964{
2965 cadence_nand_chips_cleanup(cdns_ctrl);
2966 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2967 kfree(cdns_ctrl->buf);
2968 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2969 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2970
2971 if (cdns_ctrl->dmac)
2972 dma_release_channel(cdns_ctrl->dmac);
2973}
2974
2975struct cadence_nand_dt {
2976 struct cdns_nand_ctrl cdns_ctrl;
2977 struct clk *clk;
2978};
2979
2980static const struct cadence_nand_dt_devdata cadence_nand_default = {
2981 .if_skew = 0,
2982 .has_dma = 1,
2983};
2984
2985static const struct of_device_id cadence_nand_dt_ids[] = {
2986 {
2987 .compatible = "cdns,hp-nfc",
2988 .data = &cadence_nand_default
2989 }, {}
2990};
2991
2992MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
2993
2994static int cadence_nand_dt_probe(struct platform_device *ofdev)
2995{
2996 struct resource *res;
2997 struct cadence_nand_dt *dt;
2998 struct cdns_nand_ctrl *cdns_ctrl;
2999 int ret;
3000 const struct cadence_nand_dt_devdata *devdata;
3001 u32 val;
3002
3003 devdata = device_get_match_data(&ofdev->dev);
3004 if (!devdata) {
3005 pr_err("Failed to find the right device id.\n");
3006 return -ENOMEM;
3007 }
3008
3009 dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
3010 if (!dt)
3011 return -ENOMEM;
3012
3013 cdns_ctrl = &dt->cdns_ctrl;
3014 cdns_ctrl->caps1 = devdata;
3015
3016 cdns_ctrl->dev = &ofdev->dev;
3017 cdns_ctrl->irq = platform_get_irq(ofdev, 0);
3018 if (cdns_ctrl->irq < 0)
3019 return cdns_ctrl->irq;
3020
3021 dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
3022
3023 cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
3024 if (IS_ERR(cdns_ctrl->reg))
3025 return PTR_ERR(cdns_ctrl->reg);
3026
3027 cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
3028 if (IS_ERR(cdns_ctrl->io.virt))
3029 return PTR_ERR(cdns_ctrl->io.virt);
3030 cdns_ctrl->io.dma = res->start;
3031
3032 dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
3033 if (IS_ERR(dt->clk))
3034 return PTR_ERR(dt->clk);
3035
3036 cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
3037
3038 ret = of_property_read_u32(ofdev->dev.of_node,
3039 "cdns,board-delay-ps", &val);
3040 if (ret) {
3041 val = 4830;
3042 dev_info(cdns_ctrl->dev,
3043 "missing cdns,board-delay-ps property, %d was set\n",
3044 val);
3045 }
3046 cdns_ctrl->board_delay = val;
3047
3048 ret = cadence_nand_init(cdns_ctrl);
3049 if (ret)
3050 return ret;
3051
3052 platform_set_drvdata(ofdev, dt);
3053 return 0;
3054}
3055
3056static void cadence_nand_dt_remove(struct platform_device *ofdev)
3057{
3058 struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3059
3060 cadence_nand_remove(&dt->cdns_ctrl);
3061}
3062
3063static struct platform_driver cadence_nand_dt_driver = {
3064 .probe = cadence_nand_dt_probe,
3065 .remove_new = cadence_nand_dt_remove,
3066 .driver = {
3067 .name = "cadence-nand-controller",
3068 .of_match_table = cadence_nand_dt_ids,
3069 },
3070};
3071
3072module_platform_driver(cadence_nand_dt_driver);
3073
3074MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3075MODULE_LICENSE("GPL v2");
3076MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3077
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Cadence NAND flash controller driver
4 *
5 * Copyright (C) 2019 Cadence
6 *
7 * Author: Piotr Sroka <piotrs@cadence.com>
8 */
9
10#include <linux/bitfield.h>
11#include <linux/clk.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h>
14#include <linux/interrupt.h>
15#include <linux/module.h>
16#include <linux/mtd/mtd.h>
17#include <linux/mtd/rawnand.h>
18#include <linux/iopoll.h>
19#include <linux/of.h>
20#include <linux/platform_device.h>
21#include <linux/property.h>
22#include <linux/slab.h>
23
24/*
25 * HPNFC can work in 3 modes:
26 * - PIO - can work in master or slave DMA
27 * - CDMA - needs Master DMA for accessing command descriptors.
28 * - Generic mode - can use only slave DMA.
29 * CDMA and PIO modes can be used to execute only base commands.
30 * Generic mode can be used to execute any command
31 * on NAND flash memory. Driver uses CDMA mode for
32 * block erasing, page reading, page programing.
33 * Generic mode is used for executing rest of commands.
34 */
35
36#define MAX_ADDRESS_CYC 6
37#define MAX_ERASE_ADDRESS_CYC 3
38#define MAX_DATA_SIZE 0xFFFC
39#define DMA_DATA_SIZE_ALIGN 8
40
41/* Register definition. */
42/*
43 * Command register 0.
44 * Writing data to this register will initiate a new transaction
45 * of the NF controller.
46 */
47#define CMD_REG0 0x0000
48/* Command type field mask. */
49#define CMD_REG0_CT GENMASK(31, 30)
50/* Command type CDMA. */
51#define CMD_REG0_CT_CDMA 0uL
52/* Command type generic. */
53#define CMD_REG0_CT_GEN 3uL
54/* Command thread number field mask. */
55#define CMD_REG0_TN GENMASK(27, 24)
56
57/* Command register 2. */
58#define CMD_REG2 0x0008
59/* Command register 3. */
60#define CMD_REG3 0x000C
61/* Pointer register to select which thread status will be selected. */
62#define CMD_STATUS_PTR 0x0010
63/* Command status register for selected thread. */
64#define CMD_STATUS 0x0014
65
66/* Interrupt status register. */
67#define INTR_STATUS 0x0110
68#define INTR_STATUS_SDMA_ERR BIT(22)
69#define INTR_STATUS_SDMA_TRIGG BIT(21)
70#define INTR_STATUS_UNSUPP_CMD BIT(19)
71#define INTR_STATUS_DDMA_TERR BIT(18)
72#define INTR_STATUS_CDMA_TERR BIT(17)
73#define INTR_STATUS_CDMA_IDL BIT(16)
74
75/* Interrupt enable register. */
76#define INTR_ENABLE 0x0114
77#define INTR_ENABLE_INTR_EN BIT(31)
78#define INTR_ENABLE_SDMA_ERR_EN BIT(22)
79#define INTR_ENABLE_SDMA_TRIGG_EN BIT(21)
80#define INTR_ENABLE_UNSUPP_CMD_EN BIT(19)
81#define INTR_ENABLE_DDMA_TERR_EN BIT(18)
82#define INTR_ENABLE_CDMA_TERR_EN BIT(17)
83#define INTR_ENABLE_CDMA_IDLE_EN BIT(16)
84
85/* Controller internal state. */
86#define CTRL_STATUS 0x0118
87#define CTRL_STATUS_INIT_COMP BIT(9)
88#define CTRL_STATUS_CTRL_BUSY BIT(8)
89
90/* Command Engine threads state. */
91#define TRD_STATUS 0x0120
92
93/* Command Engine interrupt thread error status. */
94#define TRD_ERR_INT_STATUS 0x0128
95/* Command Engine interrupt thread error enable. */
96#define TRD_ERR_INT_STATUS_EN 0x0130
97/* Command Engine interrupt thread complete status. */
98#define TRD_COMP_INT_STATUS 0x0138
99
100/*
101 * Transfer config 0 register.
102 * Configures data transfer parameters.
103 */
104#define TRAN_CFG_0 0x0400
105/* Offset value from the beginning of the page. */
106#define TRAN_CFG_0_OFFSET GENMASK(31, 16)
107/* Numbers of sectors to transfer within singlNF device's page. */
108#define TRAN_CFG_0_SEC_CNT GENMASK(7, 0)
109
110/*
111 * Transfer config 1 register.
112 * Configures data transfer parameters.
113 */
114#define TRAN_CFG_1 0x0404
115/* Size of last data sector. */
116#define TRAN_CFG_1_LAST_SEC_SIZE GENMASK(31, 16)
117/* Size of not-last data sector. */
118#define TRAN_CFG_1_SECTOR_SIZE GENMASK(15, 0)
119
120/* ECC engine configuration register 0. */
121#define ECC_CONFIG_0 0x0428
122/* Correction strength. */
123#define ECC_CONFIG_0_CORR_STR GENMASK(10, 8)
124/* Enable erased pages detection mechanism. */
125#define ECC_CONFIG_0_ERASE_DET_EN BIT(1)
126/* Enable controller ECC check bits generation and correction. */
127#define ECC_CONFIG_0_ECC_EN BIT(0)
128
129/* ECC engine configuration register 1. */
130#define ECC_CONFIG_1 0x042C
131
132/* Multiplane settings register. */
133#define MULTIPLANE_CFG 0x0434
134/* Cache operation settings. */
135#define CACHE_CFG 0x0438
136
137/* DMA settings register. */
138#define DMA_SETINGS 0x043C
139/* Enable SDMA error report on access unprepared slave DMA interface. */
140#define DMA_SETINGS_SDMA_ERR_RSP BIT(17)
141
142/* Transferred data block size for the slave DMA module. */
143#define SDMA_SIZE 0x0440
144
145/* Thread number associated with transferred data block
146 * for the slave DMA module.
147 */
148#define SDMA_TRD_NUM 0x0444
149/* Thread number mask. */
150#define SDMA_TRD_NUM_SDMA_TRD GENMASK(2, 0)
151
152#define CONTROL_DATA_CTRL 0x0494
153/* Thread number mask. */
154#define CONTROL_DATA_CTRL_SIZE GENMASK(15, 0)
155
156#define CTRL_VERSION 0x800
157#define CTRL_VERSION_REV GENMASK(7, 0)
158
159/* Available hardware features of the controller. */
160#define CTRL_FEATURES 0x804
161/* Support for NV-DDR2/3 work mode. */
162#define CTRL_FEATURES_NVDDR_2_3 BIT(28)
163/* Support for NV-DDR work mode. */
164#define CTRL_FEATURES_NVDDR BIT(27)
165/* Support for asynchronous work mode. */
166#define CTRL_FEATURES_ASYNC BIT(26)
167/* Support for asynchronous work mode. */
168#define CTRL_FEATURES_N_BANKS GENMASK(25, 24)
169/* Slave and Master DMA data width. */
170#define CTRL_FEATURES_DMA_DWITH64 BIT(21)
171/* Availability of Control Data feature.*/
172#define CTRL_FEATURES_CONTROL_DATA BIT(10)
173
174/* BCH Engine identification register 0 - correction strengths. */
175#define BCH_CFG_0 0x838
176#define BCH_CFG_0_CORR_CAP_0 GENMASK(7, 0)
177#define BCH_CFG_0_CORR_CAP_1 GENMASK(15, 8)
178#define BCH_CFG_0_CORR_CAP_2 GENMASK(23, 16)
179#define BCH_CFG_0_CORR_CAP_3 GENMASK(31, 24)
180
181/* BCH Engine identification register 1 - correction strengths. */
182#define BCH_CFG_1 0x83C
183#define BCH_CFG_1_CORR_CAP_4 GENMASK(7, 0)
184#define BCH_CFG_1_CORR_CAP_5 GENMASK(15, 8)
185#define BCH_CFG_1_CORR_CAP_6 GENMASK(23, 16)
186#define BCH_CFG_1_CORR_CAP_7 GENMASK(31, 24)
187
188/* BCH Engine identification register 2 - sector sizes. */
189#define BCH_CFG_2 0x840
190#define BCH_CFG_2_SECT_0 GENMASK(15, 0)
191#define BCH_CFG_2_SECT_1 GENMASK(31, 16)
192
193/* BCH Engine identification register 3. */
194#define BCH_CFG_3 0x844
195#define BCH_CFG_3_METADATA_SIZE GENMASK(23, 16)
196
197/* Ready/Busy# line status. */
198#define RBN_SETINGS 0x1004
199
200/* Common settings. */
201#define COMMON_SET 0x1008
202/* 16 bit device connected to the NAND Flash interface. */
203#define COMMON_SET_DEVICE_16BIT BIT(8)
204
205/* Skip_bytes registers. */
206#define SKIP_BYTES_CONF 0x100C
207#define SKIP_BYTES_MARKER_VALUE GENMASK(31, 16)
208#define SKIP_BYTES_NUM_OF_BYTES GENMASK(7, 0)
209
210#define SKIP_BYTES_OFFSET 0x1010
211#define SKIP_BYTES_OFFSET_VALUE GENMASK(23, 0)
212
213/* Timings configuration. */
214#define ASYNC_TOGGLE_TIMINGS 0x101c
215#define ASYNC_TOGGLE_TIMINGS_TRH GENMASK(28, 24)
216#define ASYNC_TOGGLE_TIMINGS_TRP GENMASK(20, 16)
217#define ASYNC_TOGGLE_TIMINGS_TWH GENMASK(12, 8)
218#define ASYNC_TOGGLE_TIMINGS_TWP GENMASK(4, 0)
219
220#define TIMINGS0 0x1024
221#define TIMINGS0_TADL GENMASK(31, 24)
222#define TIMINGS0_TCCS GENMASK(23, 16)
223#define TIMINGS0_TWHR GENMASK(15, 8)
224#define TIMINGS0_TRHW GENMASK(7, 0)
225
226#define TIMINGS1 0x1028
227#define TIMINGS1_TRHZ GENMASK(31, 24)
228#define TIMINGS1_TWB GENMASK(23, 16)
229#define TIMINGS1_TVDLY GENMASK(7, 0)
230
231#define TIMINGS2 0x102c
232#define TIMINGS2_TFEAT GENMASK(25, 16)
233#define TIMINGS2_CS_HOLD_TIME GENMASK(13, 8)
234#define TIMINGS2_CS_SETUP_TIME GENMASK(5, 0)
235
236/* Configuration of the resynchronization of slave DLL of PHY. */
237#define DLL_PHY_CTRL 0x1034
238#define DLL_PHY_CTRL_DLL_RST_N BIT(24)
239#define DLL_PHY_CTRL_EXTENDED_WR_MODE BIT(17)
240#define DLL_PHY_CTRL_EXTENDED_RD_MODE BIT(16)
241#define DLL_PHY_CTRL_RS_HIGH_WAIT_CNT GENMASK(11, 8)
242#define DLL_PHY_CTRL_RS_IDLE_CNT GENMASK(7, 0)
243
244/* Register controlling DQ related timing. */
245#define PHY_DQ_TIMING 0x2000
246/* Register controlling DSQ related timing. */
247#define PHY_DQS_TIMING 0x2004
248#define PHY_DQS_TIMING_DQS_SEL_OE_END GENMASK(3, 0)
249#define PHY_DQS_TIMING_PHONY_DQS_SEL BIT(16)
250#define PHY_DQS_TIMING_USE_PHONY_DQS BIT(20)
251
252/* Register controlling the gate and loopback control related timing. */
253#define PHY_GATE_LPBK_CTRL 0x2008
254#define PHY_GATE_LPBK_CTRL_RDS GENMASK(24, 19)
255
256/* Register holds the control for the master DLL logic. */
257#define PHY_DLL_MASTER_CTRL 0x200C
258#define PHY_DLL_MASTER_CTRL_BYPASS_MODE BIT(23)
259
260/* Register holds the control for the slave DLL logic. */
261#define PHY_DLL_SLAVE_CTRL 0x2010
262
263/* This register handles the global control settings for the PHY. */
264#define PHY_CTRL 0x2080
265#define PHY_CTRL_SDR_DQS BIT(14)
266#define PHY_CTRL_PHONY_DQS GENMASK(9, 4)
267
268/*
269 * This register handles the global control settings
270 * for the termination selects for reads.
271 */
272#define PHY_TSEL 0x2084
273
274/* Generic command layout. */
275#define GCMD_LAY_CS GENMASK_ULL(11, 8)
276/*
277 * This bit informs the minicotroller if it has to wait for tWB
278 * after sending the last CMD/ADDR/DATA in the sequence.
279 */
280#define GCMD_LAY_TWB BIT_ULL(6)
281/* Type of generic instruction. */
282#define GCMD_LAY_INSTR GENMASK_ULL(5, 0)
283
284/* Generic CMD sequence type. */
285#define GCMD_LAY_INSTR_CMD 0
286/* Generic ADDR sequence type. */
287#define GCMD_LAY_INSTR_ADDR 1
288/* Generic data transfer sequence type. */
289#define GCMD_LAY_INSTR_DATA 2
290
291/* Input part of generic command type of input is command. */
292#define GCMD_LAY_INPUT_CMD GENMASK_ULL(23, 16)
293
294/* Generic command address sequence - address fields. */
295#define GCMD_LAY_INPUT_ADDR GENMASK_ULL(63, 16)
296/* Generic command address sequence - address size. */
297#define GCMD_LAY_INPUT_ADDR_SIZE GENMASK_ULL(13, 11)
298
299/* Transfer direction field of generic command data sequence. */
300#define GCMD_DIR BIT_ULL(11)
301/* Read transfer direction of generic command data sequence. */
302#define GCMD_DIR_READ 0
303/* Write transfer direction of generic command data sequence. */
304#define GCMD_DIR_WRITE 1
305
306/* ECC enabled flag of generic command data sequence - ECC enabled. */
307#define GCMD_ECC_EN BIT_ULL(12)
308/* Generic command data sequence - sector size. */
309#define GCMD_SECT_SIZE GENMASK_ULL(31, 16)
310/* Generic command data sequence - sector count. */
311#define GCMD_SECT_CNT GENMASK_ULL(39, 32)
312/* Generic command data sequence - last sector size. */
313#define GCMD_LAST_SIZE GENMASK_ULL(55, 40)
314
315/* CDMA descriptor fields. */
316/* Erase command type of CDMA descriptor. */
317#define CDMA_CT_ERASE 0x1000
318/* Program page command type of CDMA descriptor. */
319#define CDMA_CT_WR 0x2100
320/* Read page command type of CDMA descriptor. */
321#define CDMA_CT_RD 0x2200
322
323/* Flash pointer memory shift. */
324#define CDMA_CFPTR_MEM_SHIFT 24
325/* Flash pointer memory mask. */
326#define CDMA_CFPTR_MEM GENMASK(26, 24)
327
328/*
329 * Command DMA descriptor flags. If set causes issue interrupt after
330 * the completion of descriptor processing.
331 */
332#define CDMA_CF_INT BIT(8)
333/*
334 * Command DMA descriptor flags - the next descriptor
335 * address field is valid and descriptor processing should continue.
336 */
337#define CDMA_CF_CONT BIT(9)
338/* DMA master flag of command DMA descriptor. */
339#define CDMA_CF_DMA_MASTER BIT(10)
340
341/* Operation complete status of command descriptor. */
342#define CDMA_CS_COMP BIT(15)
343/* Operation complete status of command descriptor. */
344/* Command descriptor status - operation fail. */
345#define CDMA_CS_FAIL BIT(14)
346/* Command descriptor status - page erased. */
347#define CDMA_CS_ERP BIT(11)
348/* Command descriptor status - timeout occurred. */
349#define CDMA_CS_TOUT BIT(10)
350/*
351 * Maximum amount of correction applied to one ECC sector.
352 * It is part of command descriptor status.
353 */
354#define CDMA_CS_MAXERR GENMASK(9, 2)
355/* Command descriptor status - uncorrectable ECC error. */
356#define CDMA_CS_UNCE BIT(1)
357/* Command descriptor status - descriptor error. */
358#define CDMA_CS_ERR BIT(0)
359
360/* Status of operation - OK. */
361#define STAT_OK 0
362/* Status of operation - FAIL. */
363#define STAT_FAIL 2
364/* Status of operation - uncorrectable ECC error. */
365#define STAT_ECC_UNCORR 3
366/* Status of operation - page erased. */
367#define STAT_ERASED 5
368/* Status of operation - correctable ECC error. */
369#define STAT_ECC_CORR 6
370/* Status of operation - unsuspected state. */
371#define STAT_UNKNOWN 7
372/* Status of operation - operation is not completed yet. */
373#define STAT_BUSY 0xFF
374
375#define BCH_MAX_NUM_CORR_CAPS 8
376#define BCH_MAX_NUM_SECTOR_SIZES 2
377
378struct cadence_nand_timings {
379 u32 async_toggle_timings;
380 u32 timings0;
381 u32 timings1;
382 u32 timings2;
383 u32 dll_phy_ctrl;
384 u32 phy_ctrl;
385 u32 phy_dqs_timing;
386 u32 phy_gate_lpbk_ctrl;
387};
388
389/* Command DMA descriptor. */
390struct cadence_nand_cdma_desc {
391 /* Next descriptor address. */
392 u64 next_pointer;
393
394 /* Flash address is a 32-bit address comprising of BANK and ROW ADDR. */
395 u32 flash_pointer;
396 /*field appears in HPNFC version 13*/
397 u16 bank;
398 u16 rsvd0;
399
400 /* Operation the controller needs to perform. */
401 u16 command_type;
402 u16 rsvd1;
403 /* Flags for operation of this command. */
404 u16 command_flags;
405 u16 rsvd2;
406
407 /* System/host memory address required for data DMA commands. */
408 u64 memory_pointer;
409
410 /* Status of operation. */
411 u32 status;
412 u32 rsvd3;
413
414 /* Address pointer to sync buffer location. */
415 u64 sync_flag_pointer;
416
417 /* Controls the buffer sync mechanism. */
418 u32 sync_arguments;
419 u32 rsvd4;
420
421 /* Control data pointer. */
422 u64 ctrl_data_ptr;
423};
424
425/* Interrupt status. */
426struct cadence_nand_irq_status {
427 /* Thread operation complete status. */
428 u32 trd_status;
429 /* Thread operation error. */
430 u32 trd_error;
431 /* Controller status. */
432 u32 status;
433};
434
435/* Cadence NAND flash controller capabilities get from driver data. */
436struct cadence_nand_dt_devdata {
437 /* Skew value of the output signals of the NAND Flash interface. */
438 u32 if_skew;
439 /* It informs if slave DMA interface is connected to DMA engine. */
440 unsigned int has_dma:1;
441};
442
443/* Cadence NAND flash controller capabilities read from registers. */
444struct cdns_nand_caps {
445 /* Maximum number of banks supported by hardware. */
446 u8 max_banks;
447 /* Slave and Master DMA data width in bytes (4 or 8). */
448 u8 data_dma_width;
449 /* Control Data feature supported. */
450 bool data_control_supp;
451 /* Is PHY type DLL. */
452 bool is_phy_type_dll;
453};
454
455struct cdns_nand_ctrl {
456 struct device *dev;
457 struct nand_controller controller;
458 struct cadence_nand_cdma_desc *cdma_desc;
459 /* IP capability. */
460 const struct cadence_nand_dt_devdata *caps1;
461 struct cdns_nand_caps caps2;
462 u8 ctrl_rev;
463 dma_addr_t dma_cdma_desc;
464 u8 *buf;
465 u32 buf_size;
466 u8 curr_corr_str_idx;
467
468 /* Register interface. */
469 void __iomem *reg;
470
471 struct {
472 void __iomem *virt;
473 dma_addr_t dma;
474 } io;
475
476 int irq;
477 /* Interrupts that have happened. */
478 struct cadence_nand_irq_status irq_status;
479 /* Interrupts we are waiting for. */
480 struct cadence_nand_irq_status irq_mask;
481 struct completion complete;
482 /* Protect irq_mask and irq_status. */
483 spinlock_t irq_lock;
484
485 int ecc_strengths[BCH_MAX_NUM_CORR_CAPS];
486 struct nand_ecc_step_info ecc_stepinfos[BCH_MAX_NUM_SECTOR_SIZES];
487 struct nand_ecc_caps ecc_caps;
488
489 int curr_trans_type;
490
491 struct dma_chan *dmac;
492
493 u32 nf_clk_rate;
494 /*
495 * Estimated Board delay. The value includes the total
496 * round trip delay for the signals and is used for deciding on values
497 * associated with data read capture.
498 */
499 u32 board_delay;
500
501 struct nand_chip *selected_chip;
502
503 unsigned long assigned_cs;
504 struct list_head chips;
505 u8 bch_metadata_size;
506};
507
508struct cdns_nand_chip {
509 struct cadence_nand_timings timings;
510 struct nand_chip chip;
511 u8 nsels;
512 struct list_head node;
513
514 /*
515 * part of oob area of NAND flash memory page.
516 * This part is available for user to read or write.
517 */
518 u32 avail_oob_size;
519
520 /* Sector size. There are few sectors per mtd->writesize */
521 u32 sector_size;
522 u32 sector_count;
523
524 /* Offset of BBM. */
525 u8 bbm_offs;
526 /* Number of bytes reserved for BBM. */
527 u8 bbm_len;
528 /* ECC strength index. */
529 u8 corr_str_idx;
530
531 u8 cs[] __counted_by(nsels);
532};
533
534struct ecc_info {
535 int (*calc_ecc_bytes)(int step_size, int strength);
536 int max_step_size;
537};
538
539static inline struct
540cdns_nand_chip *to_cdns_nand_chip(struct nand_chip *chip)
541{
542 return container_of(chip, struct cdns_nand_chip, chip);
543}
544
545static inline struct
546cdns_nand_ctrl *to_cdns_nand_ctrl(struct nand_controller *controller)
547{
548 return container_of(controller, struct cdns_nand_ctrl, controller);
549}
550
551static bool
552cadence_nand_dma_buf_ok(struct cdns_nand_ctrl *cdns_ctrl, const void *buf,
553 u32 buf_len)
554{
555 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
556
557 return buf && virt_addr_valid(buf) &&
558 likely(IS_ALIGNED((uintptr_t)buf, data_dma_width)) &&
559 likely(IS_ALIGNED(buf_len, DMA_DATA_SIZE_ALIGN));
560}
561
562static int cadence_nand_wait_for_value(struct cdns_nand_ctrl *cdns_ctrl,
563 u32 reg_offset, u32 timeout_us,
564 u32 mask, bool is_clear)
565{
566 u32 val;
567 int ret;
568
569 ret = readl_relaxed_poll_timeout(cdns_ctrl->reg + reg_offset,
570 val, !(val & mask) == is_clear,
571 10, timeout_us);
572
573 if (ret < 0) {
574 dev_err(cdns_ctrl->dev,
575 "Timeout while waiting for reg %x with mask %x is clear %d\n",
576 reg_offset, mask, is_clear);
577 }
578
579 return ret;
580}
581
582static int cadence_nand_set_ecc_enable(struct cdns_nand_ctrl *cdns_ctrl,
583 bool enable)
584{
585 u32 reg;
586
587 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
588 1000000,
589 CTRL_STATUS_CTRL_BUSY, true))
590 return -ETIMEDOUT;
591
592 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
593
594 if (enable)
595 reg |= ECC_CONFIG_0_ECC_EN;
596 else
597 reg &= ~ECC_CONFIG_0_ECC_EN;
598
599 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
600
601 return 0;
602}
603
604static void cadence_nand_set_ecc_strength(struct cdns_nand_ctrl *cdns_ctrl,
605 u8 corr_str_idx)
606{
607 u32 reg;
608
609 if (cdns_ctrl->curr_corr_str_idx == corr_str_idx)
610 return;
611
612 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
613 reg &= ~ECC_CONFIG_0_CORR_STR;
614 reg |= FIELD_PREP(ECC_CONFIG_0_CORR_STR, corr_str_idx);
615 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
616
617 cdns_ctrl->curr_corr_str_idx = corr_str_idx;
618}
619
620static int cadence_nand_get_ecc_strength_idx(struct cdns_nand_ctrl *cdns_ctrl,
621 u8 strength)
622{
623 int i, corr_str_idx = -1;
624
625 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
626 if (cdns_ctrl->ecc_strengths[i] == strength) {
627 corr_str_idx = i;
628 break;
629 }
630 }
631
632 return corr_str_idx;
633}
634
635static int cadence_nand_set_skip_marker_val(struct cdns_nand_ctrl *cdns_ctrl,
636 u16 marker_value)
637{
638 u32 reg;
639
640 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
641 1000000,
642 CTRL_STATUS_CTRL_BUSY, true))
643 return -ETIMEDOUT;
644
645 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
646 reg &= ~SKIP_BYTES_MARKER_VALUE;
647 reg |= FIELD_PREP(SKIP_BYTES_MARKER_VALUE,
648 marker_value);
649
650 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
651
652 return 0;
653}
654
655static int cadence_nand_set_skip_bytes_conf(struct cdns_nand_ctrl *cdns_ctrl,
656 u8 num_of_bytes,
657 u32 offset_value,
658 int enable)
659{
660 u32 reg, skip_bytes_offset;
661
662 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
663 1000000,
664 CTRL_STATUS_CTRL_BUSY, true))
665 return -ETIMEDOUT;
666
667 if (!enable) {
668 num_of_bytes = 0;
669 offset_value = 0;
670 }
671
672 reg = readl_relaxed(cdns_ctrl->reg + SKIP_BYTES_CONF);
673 reg &= ~SKIP_BYTES_NUM_OF_BYTES;
674 reg |= FIELD_PREP(SKIP_BYTES_NUM_OF_BYTES,
675 num_of_bytes);
676 skip_bytes_offset = FIELD_PREP(SKIP_BYTES_OFFSET_VALUE,
677 offset_value);
678
679 writel_relaxed(reg, cdns_ctrl->reg + SKIP_BYTES_CONF);
680 writel_relaxed(skip_bytes_offset, cdns_ctrl->reg + SKIP_BYTES_OFFSET);
681
682 return 0;
683}
684
685/* Functions enables/disables hardware detection of erased data */
686static void cadence_nand_set_erase_detection(struct cdns_nand_ctrl *cdns_ctrl,
687 bool enable,
688 u8 bitflips_threshold)
689{
690 u32 reg;
691
692 reg = readl_relaxed(cdns_ctrl->reg + ECC_CONFIG_0);
693
694 if (enable)
695 reg |= ECC_CONFIG_0_ERASE_DET_EN;
696 else
697 reg &= ~ECC_CONFIG_0_ERASE_DET_EN;
698
699 writel_relaxed(reg, cdns_ctrl->reg + ECC_CONFIG_0);
700 writel_relaxed(bitflips_threshold, cdns_ctrl->reg + ECC_CONFIG_1);
701}
702
703static int cadence_nand_set_access_width16(struct cdns_nand_ctrl *cdns_ctrl,
704 bool bit_bus16)
705{
706 u32 reg;
707
708 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
709 1000000,
710 CTRL_STATUS_CTRL_BUSY, true))
711 return -ETIMEDOUT;
712
713 reg = readl_relaxed(cdns_ctrl->reg + COMMON_SET);
714
715 if (!bit_bus16)
716 reg &= ~COMMON_SET_DEVICE_16BIT;
717 else
718 reg |= COMMON_SET_DEVICE_16BIT;
719 writel_relaxed(reg, cdns_ctrl->reg + COMMON_SET);
720
721 return 0;
722}
723
724static void
725cadence_nand_clear_interrupt(struct cdns_nand_ctrl *cdns_ctrl,
726 struct cadence_nand_irq_status *irq_status)
727{
728 writel_relaxed(irq_status->status, cdns_ctrl->reg + INTR_STATUS);
729 writel_relaxed(irq_status->trd_status,
730 cdns_ctrl->reg + TRD_COMP_INT_STATUS);
731 writel_relaxed(irq_status->trd_error,
732 cdns_ctrl->reg + TRD_ERR_INT_STATUS);
733}
734
735static void
736cadence_nand_read_int_status(struct cdns_nand_ctrl *cdns_ctrl,
737 struct cadence_nand_irq_status *irq_status)
738{
739 irq_status->status = readl_relaxed(cdns_ctrl->reg + INTR_STATUS);
740 irq_status->trd_status = readl_relaxed(cdns_ctrl->reg
741 + TRD_COMP_INT_STATUS);
742 irq_status->trd_error = readl_relaxed(cdns_ctrl->reg
743 + TRD_ERR_INT_STATUS);
744}
745
746static u32 irq_detected(struct cdns_nand_ctrl *cdns_ctrl,
747 struct cadence_nand_irq_status *irq_status)
748{
749 cadence_nand_read_int_status(cdns_ctrl, irq_status);
750
751 return irq_status->status || irq_status->trd_status ||
752 irq_status->trd_error;
753}
754
755static void cadence_nand_reset_irq(struct cdns_nand_ctrl *cdns_ctrl)
756{
757 unsigned long flags;
758
759 spin_lock_irqsave(&cdns_ctrl->irq_lock, flags);
760 memset(&cdns_ctrl->irq_status, 0, sizeof(cdns_ctrl->irq_status));
761 memset(&cdns_ctrl->irq_mask, 0, sizeof(cdns_ctrl->irq_mask));
762 spin_unlock_irqrestore(&cdns_ctrl->irq_lock, flags);
763}
764
765/*
766 * This is the interrupt service routine. It handles all interrupts
767 * sent to this device.
768 */
769static irqreturn_t cadence_nand_isr(int irq, void *dev_id)
770{
771 struct cdns_nand_ctrl *cdns_ctrl = dev_id;
772 struct cadence_nand_irq_status irq_status;
773 irqreturn_t result = IRQ_NONE;
774
775 spin_lock(&cdns_ctrl->irq_lock);
776
777 if (irq_detected(cdns_ctrl, &irq_status)) {
778 /* Handle interrupt. */
779 /* First acknowledge it. */
780 cadence_nand_clear_interrupt(cdns_ctrl, &irq_status);
781 /* Status in the device context for someone to read. */
782 cdns_ctrl->irq_status.status |= irq_status.status;
783 cdns_ctrl->irq_status.trd_status |= irq_status.trd_status;
784 cdns_ctrl->irq_status.trd_error |= irq_status.trd_error;
785 /* Notify anyone who cares that it happened. */
786 complete(&cdns_ctrl->complete);
787 /* Tell the OS that we've handled this. */
788 result = IRQ_HANDLED;
789 }
790 spin_unlock(&cdns_ctrl->irq_lock);
791
792 return result;
793}
794
795static void cadence_nand_set_irq_mask(struct cdns_nand_ctrl *cdns_ctrl,
796 struct cadence_nand_irq_status *irq_mask)
797{
798 writel_relaxed(INTR_ENABLE_INTR_EN | irq_mask->status,
799 cdns_ctrl->reg + INTR_ENABLE);
800
801 writel_relaxed(irq_mask->trd_error,
802 cdns_ctrl->reg + TRD_ERR_INT_STATUS_EN);
803}
804
805static void
806cadence_nand_wait_for_irq(struct cdns_nand_ctrl *cdns_ctrl,
807 struct cadence_nand_irq_status *irq_mask,
808 struct cadence_nand_irq_status *irq_status)
809{
810 unsigned long timeout = msecs_to_jiffies(10000);
811 unsigned long time_left;
812
813 time_left = wait_for_completion_timeout(&cdns_ctrl->complete,
814 timeout);
815
816 *irq_status = cdns_ctrl->irq_status;
817 if (time_left == 0) {
818 /* Timeout error. */
819 dev_err(cdns_ctrl->dev, "timeout occurred:\n");
820 dev_err(cdns_ctrl->dev, "\tstatus = 0x%x, mask = 0x%x\n",
821 irq_status->status, irq_mask->status);
822 dev_err(cdns_ctrl->dev,
823 "\ttrd_status = 0x%x, trd_status mask = 0x%x\n",
824 irq_status->trd_status, irq_mask->trd_status);
825 dev_err(cdns_ctrl->dev,
826 "\t trd_error = 0x%x, trd_error mask = 0x%x\n",
827 irq_status->trd_error, irq_mask->trd_error);
828 }
829}
830
831/* Execute generic command on NAND controller. */
832static int cadence_nand_generic_cmd_send(struct cdns_nand_ctrl *cdns_ctrl,
833 u8 chip_nr,
834 u64 mini_ctrl_cmd)
835{
836 u32 mini_ctrl_cmd_l, mini_ctrl_cmd_h, reg;
837
838 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_CS, chip_nr);
839 mini_ctrl_cmd_l = mini_ctrl_cmd & 0xFFFFFFFF;
840 mini_ctrl_cmd_h = mini_ctrl_cmd >> 32;
841
842 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
843 1000000,
844 CTRL_STATUS_CTRL_BUSY, true))
845 return -ETIMEDOUT;
846
847 cadence_nand_reset_irq(cdns_ctrl);
848
849 writel_relaxed(mini_ctrl_cmd_l, cdns_ctrl->reg + CMD_REG2);
850 writel_relaxed(mini_ctrl_cmd_h, cdns_ctrl->reg + CMD_REG3);
851
852 /* Select generic command. */
853 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_GEN);
854 /* Thread number. */
855 reg |= FIELD_PREP(CMD_REG0_TN, 0);
856
857 /* Issue command. */
858 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
859
860 return 0;
861}
862
863/* Wait for data on slave DMA interface. */
864static int cadence_nand_wait_on_sdma(struct cdns_nand_ctrl *cdns_ctrl,
865 u8 *out_sdma_trd,
866 u32 *out_sdma_size)
867{
868 struct cadence_nand_irq_status irq_mask, irq_status;
869
870 irq_mask.trd_status = 0;
871 irq_mask.trd_error = 0;
872 irq_mask.status = INTR_STATUS_SDMA_TRIGG
873 | INTR_STATUS_SDMA_ERR
874 | INTR_STATUS_UNSUPP_CMD;
875
876 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
877 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
878 if (irq_status.status == 0) {
879 dev_err(cdns_ctrl->dev, "Timeout while waiting for SDMA\n");
880 return -ETIMEDOUT;
881 }
882
883 if (irq_status.status & INTR_STATUS_SDMA_TRIGG) {
884 *out_sdma_size = readl_relaxed(cdns_ctrl->reg + SDMA_SIZE);
885 *out_sdma_trd = readl_relaxed(cdns_ctrl->reg + SDMA_TRD_NUM);
886 *out_sdma_trd =
887 FIELD_GET(SDMA_TRD_NUM_SDMA_TRD, *out_sdma_trd);
888 } else {
889 dev_err(cdns_ctrl->dev, "SDMA error - irq_status %x\n",
890 irq_status.status);
891 return -EIO;
892 }
893
894 return 0;
895}
896
897static void cadence_nand_get_caps(struct cdns_nand_ctrl *cdns_ctrl)
898{
899 u32 reg;
900
901 reg = readl_relaxed(cdns_ctrl->reg + CTRL_FEATURES);
902
903 cdns_ctrl->caps2.max_banks = 1 << FIELD_GET(CTRL_FEATURES_N_BANKS, reg);
904
905 if (FIELD_GET(CTRL_FEATURES_DMA_DWITH64, reg))
906 cdns_ctrl->caps2.data_dma_width = 8;
907 else
908 cdns_ctrl->caps2.data_dma_width = 4;
909
910 if (reg & CTRL_FEATURES_CONTROL_DATA)
911 cdns_ctrl->caps2.data_control_supp = true;
912
913 if (reg & (CTRL_FEATURES_NVDDR_2_3
914 | CTRL_FEATURES_NVDDR))
915 cdns_ctrl->caps2.is_phy_type_dll = true;
916}
917
918/* Prepare CDMA descriptor. */
919static void
920cadence_nand_cdma_desc_prepare(struct cdns_nand_ctrl *cdns_ctrl,
921 char nf_mem, u32 flash_ptr, dma_addr_t mem_ptr,
922 dma_addr_t ctrl_data_ptr, u16 ctype)
923{
924 struct cadence_nand_cdma_desc *cdma_desc = cdns_ctrl->cdma_desc;
925
926 memset(cdma_desc, 0, sizeof(struct cadence_nand_cdma_desc));
927
928 /* Set fields for one descriptor. */
929 cdma_desc->flash_pointer = flash_ptr;
930 if (cdns_ctrl->ctrl_rev >= 13)
931 cdma_desc->bank = nf_mem;
932 else
933 cdma_desc->flash_pointer |= (nf_mem << CDMA_CFPTR_MEM_SHIFT);
934
935 cdma_desc->command_flags |= CDMA_CF_DMA_MASTER;
936 cdma_desc->command_flags |= CDMA_CF_INT;
937
938 cdma_desc->memory_pointer = mem_ptr;
939 cdma_desc->status = 0;
940 cdma_desc->sync_flag_pointer = 0;
941 cdma_desc->sync_arguments = 0;
942
943 cdma_desc->command_type = ctype;
944 cdma_desc->ctrl_data_ptr = ctrl_data_ptr;
945}
946
947static u8 cadence_nand_check_desc_error(struct cdns_nand_ctrl *cdns_ctrl,
948 u32 desc_status)
949{
950 if (desc_status & CDMA_CS_ERP)
951 return STAT_ERASED;
952
953 if (desc_status & CDMA_CS_UNCE)
954 return STAT_ECC_UNCORR;
955
956 if (desc_status & CDMA_CS_ERR) {
957 dev_err(cdns_ctrl->dev, ":CDMA desc error flag detected.\n");
958 return STAT_FAIL;
959 }
960
961 if (FIELD_GET(CDMA_CS_MAXERR, desc_status))
962 return STAT_ECC_CORR;
963
964 return STAT_FAIL;
965}
966
967static int cadence_nand_cdma_finish(struct cdns_nand_ctrl *cdns_ctrl)
968{
969 struct cadence_nand_cdma_desc *desc_ptr = cdns_ctrl->cdma_desc;
970 u8 status = STAT_BUSY;
971
972 if (desc_ptr->status & CDMA_CS_FAIL) {
973 status = cadence_nand_check_desc_error(cdns_ctrl,
974 desc_ptr->status);
975 dev_err(cdns_ctrl->dev, ":CDMA error %x\n", desc_ptr->status);
976 } else if (desc_ptr->status & CDMA_CS_COMP) {
977 /* Descriptor finished with no errors. */
978 if (desc_ptr->command_flags & CDMA_CF_CONT) {
979 dev_info(cdns_ctrl->dev, "DMA unsupported flag is set");
980 status = STAT_UNKNOWN;
981 } else {
982 /* Last descriptor. */
983 status = STAT_OK;
984 }
985 }
986
987 return status;
988}
989
990static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl,
991 u8 thread)
992{
993 u32 reg;
994 int status;
995
996 /* Wait for thread ready. */
997 status = cadence_nand_wait_for_value(cdns_ctrl, TRD_STATUS,
998 1000000,
999 BIT(thread), true);
1000 if (status)
1001 return status;
1002
1003 cadence_nand_reset_irq(cdns_ctrl);
1004 reinit_completion(&cdns_ctrl->complete);
1005
1006 writel_relaxed((u32)cdns_ctrl->dma_cdma_desc,
1007 cdns_ctrl->reg + CMD_REG2);
1008 writel_relaxed(0, cdns_ctrl->reg + CMD_REG3);
1009
1010 /* Select CDMA mode. */
1011 reg = FIELD_PREP(CMD_REG0_CT, CMD_REG0_CT_CDMA);
1012 /* Thread number. */
1013 reg |= FIELD_PREP(CMD_REG0_TN, thread);
1014 /* Issue command. */
1015 writel_relaxed(reg, cdns_ctrl->reg + CMD_REG0);
1016
1017 return 0;
1018}
1019
1020/* Send SDMA command and wait for finish. */
1021static u32
1022cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl,
1023 u8 thread)
1024{
1025 struct cadence_nand_irq_status irq_mask, irq_status = {0};
1026 int status;
1027
1028 irq_mask.trd_status = BIT(thread);
1029 irq_mask.trd_error = BIT(thread);
1030 irq_mask.status = INTR_STATUS_CDMA_TERR;
1031
1032 cadence_nand_set_irq_mask(cdns_ctrl, &irq_mask);
1033
1034 status = cadence_nand_cdma_send(cdns_ctrl, thread);
1035 if (status)
1036 return status;
1037
1038 cadence_nand_wait_for_irq(cdns_ctrl, &irq_mask, &irq_status);
1039
1040 if (irq_status.status == 0 && irq_status.trd_status == 0 &&
1041 irq_status.trd_error == 0) {
1042 dev_err(cdns_ctrl->dev, "CDMA command timeout\n");
1043 return -ETIMEDOUT;
1044 }
1045 if (irq_status.status & irq_mask.status) {
1046 dev_err(cdns_ctrl->dev, "CDMA command failed\n");
1047 return -EIO;
1048 }
1049
1050 return 0;
1051}
1052
1053/*
1054 * ECC size depends on configured ECC strength and on maximum supported
1055 * ECC step size.
1056 */
1057static int cadence_nand_calc_ecc_bytes(int max_step_size, int strength)
1058{
1059 int nbytes = DIV_ROUND_UP(fls(8 * max_step_size) * strength, 8);
1060
1061 return ALIGN(nbytes, 2);
1062}
1063
1064#define CADENCE_NAND_CALC_ECC_BYTES(max_step_size) \
1065 static int \
1066 cadence_nand_calc_ecc_bytes_##max_step_size(int step_size, \
1067 int strength)\
1068 {\
1069 return cadence_nand_calc_ecc_bytes(max_step_size, strength);\
1070 }
1071
1072CADENCE_NAND_CALC_ECC_BYTES(256)
1073CADENCE_NAND_CALC_ECC_BYTES(512)
1074CADENCE_NAND_CALC_ECC_BYTES(1024)
1075CADENCE_NAND_CALC_ECC_BYTES(2048)
1076CADENCE_NAND_CALC_ECC_BYTES(4096)
1077
1078/* Function reads BCH capabilities. */
1079static int cadence_nand_read_bch_caps(struct cdns_nand_ctrl *cdns_ctrl)
1080{
1081 struct nand_ecc_caps *ecc_caps = &cdns_ctrl->ecc_caps;
1082 int max_step_size = 0, nstrengths, i;
1083 u32 reg;
1084
1085 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_3);
1086 cdns_ctrl->bch_metadata_size = FIELD_GET(BCH_CFG_3_METADATA_SIZE, reg);
1087 if (cdns_ctrl->bch_metadata_size < 4) {
1088 dev_err(cdns_ctrl->dev,
1089 "Driver needs at least 4 bytes of BCH meta data\n");
1090 return -EIO;
1091 }
1092
1093 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_0);
1094 cdns_ctrl->ecc_strengths[0] = FIELD_GET(BCH_CFG_0_CORR_CAP_0, reg);
1095 cdns_ctrl->ecc_strengths[1] = FIELD_GET(BCH_CFG_0_CORR_CAP_1, reg);
1096 cdns_ctrl->ecc_strengths[2] = FIELD_GET(BCH_CFG_0_CORR_CAP_2, reg);
1097 cdns_ctrl->ecc_strengths[3] = FIELD_GET(BCH_CFG_0_CORR_CAP_3, reg);
1098
1099 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_1);
1100 cdns_ctrl->ecc_strengths[4] = FIELD_GET(BCH_CFG_1_CORR_CAP_4, reg);
1101 cdns_ctrl->ecc_strengths[5] = FIELD_GET(BCH_CFG_1_CORR_CAP_5, reg);
1102 cdns_ctrl->ecc_strengths[6] = FIELD_GET(BCH_CFG_1_CORR_CAP_6, reg);
1103 cdns_ctrl->ecc_strengths[7] = FIELD_GET(BCH_CFG_1_CORR_CAP_7, reg);
1104
1105 reg = readl_relaxed(cdns_ctrl->reg + BCH_CFG_2);
1106 cdns_ctrl->ecc_stepinfos[0].stepsize =
1107 FIELD_GET(BCH_CFG_2_SECT_0, reg);
1108
1109 cdns_ctrl->ecc_stepinfos[1].stepsize =
1110 FIELD_GET(BCH_CFG_2_SECT_1, reg);
1111
1112 nstrengths = 0;
1113 for (i = 0; i < BCH_MAX_NUM_CORR_CAPS; i++) {
1114 if (cdns_ctrl->ecc_strengths[i] != 0)
1115 nstrengths++;
1116 }
1117
1118 ecc_caps->nstepinfos = 0;
1119 for (i = 0; i < BCH_MAX_NUM_SECTOR_SIZES; i++) {
1120 /* ECC strengths are common for all step infos. */
1121 cdns_ctrl->ecc_stepinfos[i].nstrengths = nstrengths;
1122 cdns_ctrl->ecc_stepinfos[i].strengths =
1123 cdns_ctrl->ecc_strengths;
1124
1125 if (cdns_ctrl->ecc_stepinfos[i].stepsize != 0)
1126 ecc_caps->nstepinfos++;
1127
1128 if (cdns_ctrl->ecc_stepinfos[i].stepsize > max_step_size)
1129 max_step_size = cdns_ctrl->ecc_stepinfos[i].stepsize;
1130 }
1131 ecc_caps->stepinfos = &cdns_ctrl->ecc_stepinfos[0];
1132
1133 switch (max_step_size) {
1134 case 256:
1135 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_256;
1136 break;
1137 case 512:
1138 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_512;
1139 break;
1140 case 1024:
1141 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_1024;
1142 break;
1143 case 2048:
1144 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_2048;
1145 break;
1146 case 4096:
1147 ecc_caps->calc_ecc_bytes = &cadence_nand_calc_ecc_bytes_4096;
1148 break;
1149 default:
1150 dev_err(cdns_ctrl->dev,
1151 "Unsupported sector size(ecc step size) %d\n",
1152 max_step_size);
1153 return -EIO;
1154 }
1155
1156 return 0;
1157}
1158
1159/* Hardware initialization. */
1160static int cadence_nand_hw_init(struct cdns_nand_ctrl *cdns_ctrl)
1161{
1162 int status;
1163 u32 reg;
1164
1165 status = cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1166 1000000,
1167 CTRL_STATUS_INIT_COMP, false);
1168 if (status)
1169 return status;
1170
1171 reg = readl_relaxed(cdns_ctrl->reg + CTRL_VERSION);
1172 cdns_ctrl->ctrl_rev = FIELD_GET(CTRL_VERSION_REV, reg);
1173
1174 dev_info(cdns_ctrl->dev,
1175 "%s: cadence nand controller version reg %x\n",
1176 __func__, reg);
1177
1178 /* Disable cache and multiplane. */
1179 writel_relaxed(0, cdns_ctrl->reg + MULTIPLANE_CFG);
1180 writel_relaxed(0, cdns_ctrl->reg + CACHE_CFG);
1181
1182 /* Clear all interrupts. */
1183 writel_relaxed(0xFFFFFFFF, cdns_ctrl->reg + INTR_STATUS);
1184
1185 cadence_nand_get_caps(cdns_ctrl);
1186 if (cadence_nand_read_bch_caps(cdns_ctrl))
1187 return -EIO;
1188
1189#ifndef CONFIG_64BIT
1190 if (cdns_ctrl->caps2.data_dma_width == 8) {
1191 dev_err(cdns_ctrl->dev,
1192 "cannot access 64-bit dma on !64-bit architectures");
1193 return -EIO;
1194 }
1195#endif
1196
1197 /*
1198 * Set IO width access to 8.
1199 * It is because during SW device discovering width access
1200 * is expected to be 8.
1201 */
1202 status = cadence_nand_set_access_width16(cdns_ctrl, false);
1203
1204 return status;
1205}
1206
1207#define TT_MAIN_OOB_AREAS 2
1208#define TT_RAW_PAGE 3
1209#define TT_BBM 4
1210#define TT_MAIN_OOB_AREA_EXT 5
1211
1212/* Prepare size of data to transfer. */
1213static void
1214cadence_nand_prepare_data_size(struct nand_chip *chip,
1215 int transfer_type)
1216{
1217 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1218 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1219 struct mtd_info *mtd = nand_to_mtd(chip);
1220 u32 sec_size = 0, offset = 0, sec_cnt = 1;
1221 u32 last_sec_size = cdns_chip->sector_size;
1222 u32 data_ctrl_size = 0;
1223 u32 reg = 0;
1224
1225 if (cdns_ctrl->curr_trans_type == transfer_type)
1226 return;
1227
1228 switch (transfer_type) {
1229 case TT_MAIN_OOB_AREA_EXT:
1230 sec_cnt = cdns_chip->sector_count;
1231 sec_size = cdns_chip->sector_size;
1232 data_ctrl_size = cdns_chip->avail_oob_size;
1233 break;
1234 case TT_MAIN_OOB_AREAS:
1235 sec_cnt = cdns_chip->sector_count;
1236 last_sec_size = cdns_chip->sector_size
1237 + cdns_chip->avail_oob_size;
1238 sec_size = cdns_chip->sector_size;
1239 break;
1240 case TT_RAW_PAGE:
1241 last_sec_size = mtd->writesize + mtd->oobsize;
1242 break;
1243 case TT_BBM:
1244 offset = mtd->writesize + cdns_chip->bbm_offs;
1245 last_sec_size = 8;
1246 break;
1247 }
1248
1249 reg = 0;
1250 reg |= FIELD_PREP(TRAN_CFG_0_OFFSET, offset);
1251 reg |= FIELD_PREP(TRAN_CFG_0_SEC_CNT, sec_cnt);
1252 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_0);
1253
1254 reg = 0;
1255 reg |= FIELD_PREP(TRAN_CFG_1_LAST_SEC_SIZE, last_sec_size);
1256 reg |= FIELD_PREP(TRAN_CFG_1_SECTOR_SIZE, sec_size);
1257 writel_relaxed(reg, cdns_ctrl->reg + TRAN_CFG_1);
1258
1259 if (cdns_ctrl->caps2.data_control_supp) {
1260 reg = readl_relaxed(cdns_ctrl->reg + CONTROL_DATA_CTRL);
1261 reg &= ~CONTROL_DATA_CTRL_SIZE;
1262 reg |= FIELD_PREP(CONTROL_DATA_CTRL_SIZE, data_ctrl_size);
1263 writel_relaxed(reg, cdns_ctrl->reg + CONTROL_DATA_CTRL);
1264 }
1265
1266 cdns_ctrl->curr_trans_type = transfer_type;
1267}
1268
1269static int
1270cadence_nand_cdma_transfer(struct cdns_nand_ctrl *cdns_ctrl, u8 chip_nr,
1271 int page, void *buf, void *ctrl_dat, u32 buf_size,
1272 u32 ctrl_dat_size, enum dma_data_direction dir,
1273 bool with_ecc)
1274{
1275 dma_addr_t dma_buf, dma_ctrl_dat = 0;
1276 u8 thread_nr = chip_nr;
1277 int status;
1278 u16 ctype;
1279
1280 if (dir == DMA_FROM_DEVICE)
1281 ctype = CDMA_CT_RD;
1282 else
1283 ctype = CDMA_CT_WR;
1284
1285 cadence_nand_set_ecc_enable(cdns_ctrl, with_ecc);
1286
1287 dma_buf = dma_map_single(cdns_ctrl->dev, buf, buf_size, dir);
1288 if (dma_mapping_error(cdns_ctrl->dev, dma_buf)) {
1289 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1290 return -EIO;
1291 }
1292
1293 if (ctrl_dat && ctrl_dat_size) {
1294 dma_ctrl_dat = dma_map_single(cdns_ctrl->dev, ctrl_dat,
1295 ctrl_dat_size, dir);
1296 if (dma_mapping_error(cdns_ctrl->dev, dma_ctrl_dat)) {
1297 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1298 buf_size, dir);
1299 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1300 return -EIO;
1301 }
1302 }
1303
1304 cadence_nand_cdma_desc_prepare(cdns_ctrl, chip_nr, page,
1305 dma_buf, dma_ctrl_dat, ctype);
1306
1307 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1308
1309 dma_unmap_single(cdns_ctrl->dev, dma_buf,
1310 buf_size, dir);
1311
1312 if (ctrl_dat && ctrl_dat_size)
1313 dma_unmap_single(cdns_ctrl->dev, dma_ctrl_dat,
1314 ctrl_dat_size, dir);
1315 if (status)
1316 return status;
1317
1318 return cadence_nand_cdma_finish(cdns_ctrl);
1319}
1320
1321static void cadence_nand_set_timings(struct cdns_nand_ctrl *cdns_ctrl,
1322 struct cadence_nand_timings *t)
1323{
1324 writel_relaxed(t->async_toggle_timings,
1325 cdns_ctrl->reg + ASYNC_TOGGLE_TIMINGS);
1326 writel_relaxed(t->timings0, cdns_ctrl->reg + TIMINGS0);
1327 writel_relaxed(t->timings1, cdns_ctrl->reg + TIMINGS1);
1328 writel_relaxed(t->timings2, cdns_ctrl->reg + TIMINGS2);
1329
1330 if (cdns_ctrl->caps2.is_phy_type_dll)
1331 writel_relaxed(t->dll_phy_ctrl, cdns_ctrl->reg + DLL_PHY_CTRL);
1332
1333 writel_relaxed(t->phy_ctrl, cdns_ctrl->reg + PHY_CTRL);
1334
1335 if (cdns_ctrl->caps2.is_phy_type_dll) {
1336 writel_relaxed(0, cdns_ctrl->reg + PHY_TSEL);
1337 writel_relaxed(2, cdns_ctrl->reg + PHY_DQ_TIMING);
1338 writel_relaxed(t->phy_dqs_timing,
1339 cdns_ctrl->reg + PHY_DQS_TIMING);
1340 writel_relaxed(t->phy_gate_lpbk_ctrl,
1341 cdns_ctrl->reg + PHY_GATE_LPBK_CTRL);
1342 writel_relaxed(PHY_DLL_MASTER_CTRL_BYPASS_MODE,
1343 cdns_ctrl->reg + PHY_DLL_MASTER_CTRL);
1344 writel_relaxed(0, cdns_ctrl->reg + PHY_DLL_SLAVE_CTRL);
1345 }
1346}
1347
1348static int cadence_nand_select_target(struct nand_chip *chip)
1349{
1350 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1351 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1352
1353 if (chip == cdns_ctrl->selected_chip)
1354 return 0;
1355
1356 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
1357 1000000,
1358 CTRL_STATUS_CTRL_BUSY, true))
1359 return -ETIMEDOUT;
1360
1361 cadence_nand_set_timings(cdns_ctrl, &cdns_chip->timings);
1362
1363 cadence_nand_set_ecc_strength(cdns_ctrl,
1364 cdns_chip->corr_str_idx);
1365
1366 cadence_nand_set_erase_detection(cdns_ctrl, true,
1367 chip->ecc.strength);
1368
1369 cdns_ctrl->curr_trans_type = -1;
1370 cdns_ctrl->selected_chip = chip;
1371
1372 return 0;
1373}
1374
1375static int cadence_nand_erase(struct nand_chip *chip, u32 page)
1376{
1377 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1378 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1379 int status;
1380 u8 thread_nr = cdns_chip->cs[chip->cur_cs];
1381
1382 cadence_nand_cdma_desc_prepare(cdns_ctrl,
1383 cdns_chip->cs[chip->cur_cs],
1384 page, 0, 0,
1385 CDMA_CT_ERASE);
1386 status = cadence_nand_cdma_send_and_wait(cdns_ctrl, thread_nr);
1387 if (status) {
1388 dev_err(cdns_ctrl->dev, "erase operation failed\n");
1389 return -EIO;
1390 }
1391
1392 status = cadence_nand_cdma_finish(cdns_ctrl);
1393 if (status)
1394 return status;
1395
1396 return 0;
1397}
1398
1399static int cadence_nand_read_bbm(struct nand_chip *chip, int page, u8 *buf)
1400{
1401 int status;
1402 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1403 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1404 struct mtd_info *mtd = nand_to_mtd(chip);
1405
1406 cadence_nand_prepare_data_size(chip, TT_BBM);
1407
1408 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1409
1410 /*
1411 * Read only bad block marker from offset
1412 * defined by a memory manufacturer.
1413 */
1414 status = cadence_nand_cdma_transfer(cdns_ctrl,
1415 cdns_chip->cs[chip->cur_cs],
1416 page, cdns_ctrl->buf, NULL,
1417 mtd->oobsize,
1418 0, DMA_FROM_DEVICE, false);
1419 if (status) {
1420 dev_err(cdns_ctrl->dev, "read BBM failed\n");
1421 return -EIO;
1422 }
1423
1424 memcpy(buf + cdns_chip->bbm_offs, cdns_ctrl->buf, cdns_chip->bbm_len);
1425
1426 return 0;
1427}
1428
1429static int cadence_nand_write_page(struct nand_chip *chip,
1430 const u8 *buf, int oob_required,
1431 int page)
1432{
1433 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1434 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1435 struct mtd_info *mtd = nand_to_mtd(chip);
1436 int status;
1437 u16 marker_val = 0xFFFF;
1438
1439 status = cadence_nand_select_target(chip);
1440 if (status)
1441 return status;
1442
1443 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1444 mtd->writesize
1445 + cdns_chip->bbm_offs,
1446 1);
1447
1448 if (oob_required) {
1449 marker_val = *(u16 *)(chip->oob_poi
1450 + cdns_chip->bbm_offs);
1451 } else {
1452 /* Set oob data to 0xFF. */
1453 memset(cdns_ctrl->buf + mtd->writesize, 0xFF,
1454 cdns_chip->avail_oob_size);
1455 }
1456
1457 cadence_nand_set_skip_marker_val(cdns_ctrl, marker_val);
1458
1459 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1460
1461 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1462 cdns_ctrl->caps2.data_control_supp) {
1463 u8 *oob;
1464
1465 if (oob_required)
1466 oob = chip->oob_poi;
1467 else
1468 oob = cdns_ctrl->buf + mtd->writesize;
1469
1470 status = cadence_nand_cdma_transfer(cdns_ctrl,
1471 cdns_chip->cs[chip->cur_cs],
1472 page, (void *)buf, oob,
1473 mtd->writesize,
1474 cdns_chip->avail_oob_size,
1475 DMA_TO_DEVICE, true);
1476 if (status) {
1477 dev_err(cdns_ctrl->dev, "write page failed\n");
1478 return -EIO;
1479 }
1480
1481 return 0;
1482 }
1483
1484 if (oob_required) {
1485 /* Transfer the data to the oob area. */
1486 memcpy(cdns_ctrl->buf + mtd->writesize, chip->oob_poi,
1487 cdns_chip->avail_oob_size);
1488 }
1489
1490 memcpy(cdns_ctrl->buf, buf, mtd->writesize);
1491
1492 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1493
1494 return cadence_nand_cdma_transfer(cdns_ctrl,
1495 cdns_chip->cs[chip->cur_cs],
1496 page, cdns_ctrl->buf, NULL,
1497 mtd->writesize
1498 + cdns_chip->avail_oob_size,
1499 0, DMA_TO_DEVICE, true);
1500}
1501
1502static int cadence_nand_write_oob(struct nand_chip *chip, int page)
1503{
1504 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1505 struct mtd_info *mtd = nand_to_mtd(chip);
1506
1507 memset(cdns_ctrl->buf, 0xFF, mtd->writesize);
1508
1509 return cadence_nand_write_page(chip, cdns_ctrl->buf, 1, page);
1510}
1511
1512static int cadence_nand_write_page_raw(struct nand_chip *chip,
1513 const u8 *buf, int oob_required,
1514 int page)
1515{
1516 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1517 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1518 struct mtd_info *mtd = nand_to_mtd(chip);
1519 int writesize = mtd->writesize;
1520 int oobsize = mtd->oobsize;
1521 int ecc_steps = chip->ecc.steps;
1522 int ecc_size = chip->ecc.size;
1523 int ecc_bytes = chip->ecc.bytes;
1524 void *tmp_buf = cdns_ctrl->buf;
1525 int oob_skip = cdns_chip->bbm_len;
1526 size_t size = writesize + oobsize;
1527 int i, pos, len;
1528 int status = 0;
1529
1530 status = cadence_nand_select_target(chip);
1531 if (status)
1532 return status;
1533
1534 /*
1535 * Fill the buffer with 0xff first except the full page transfer.
1536 * This simplifies the logic.
1537 */
1538 if (!buf || !oob_required)
1539 memset(tmp_buf, 0xff, size);
1540
1541 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1542
1543 /* Arrange the buffer for syndrome payload/ecc layout. */
1544 if (buf) {
1545 for (i = 0; i < ecc_steps; i++) {
1546 pos = i * (ecc_size + ecc_bytes);
1547 len = ecc_size;
1548
1549 if (pos >= writesize)
1550 pos += oob_skip;
1551 else if (pos + len > writesize)
1552 len = writesize - pos;
1553
1554 memcpy(tmp_buf + pos, buf, len);
1555 buf += len;
1556 if (len < ecc_size) {
1557 len = ecc_size - len;
1558 memcpy(tmp_buf + writesize + oob_skip, buf,
1559 len);
1560 buf += len;
1561 }
1562 }
1563 }
1564
1565 if (oob_required) {
1566 const u8 *oob = chip->oob_poi;
1567 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1568 (cdns_chip->sector_size + chip->ecc.bytes)
1569 + cdns_chip->sector_size + oob_skip;
1570
1571 /* BBM at the beginning of the OOB area. */
1572 memcpy(tmp_buf + writesize, oob, oob_skip);
1573
1574 /* OOB free. */
1575 memcpy(tmp_buf + oob_data_offset, oob,
1576 cdns_chip->avail_oob_size);
1577 oob += cdns_chip->avail_oob_size;
1578
1579 /* OOB ECC. */
1580 for (i = 0; i < ecc_steps; i++) {
1581 pos = ecc_size + i * (ecc_size + ecc_bytes);
1582 if (i == (ecc_steps - 1))
1583 pos += cdns_chip->avail_oob_size;
1584
1585 len = ecc_bytes;
1586
1587 if (pos >= writesize)
1588 pos += oob_skip;
1589 else if (pos + len > writesize)
1590 len = writesize - pos;
1591
1592 memcpy(tmp_buf + pos, oob, len);
1593 oob += len;
1594 if (len < ecc_bytes) {
1595 len = ecc_bytes - len;
1596 memcpy(tmp_buf + writesize + oob_skip, oob,
1597 len);
1598 oob += len;
1599 }
1600 }
1601 }
1602
1603 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1604
1605 return cadence_nand_cdma_transfer(cdns_ctrl,
1606 cdns_chip->cs[chip->cur_cs],
1607 page, cdns_ctrl->buf, NULL,
1608 mtd->writesize +
1609 mtd->oobsize,
1610 0, DMA_TO_DEVICE, false);
1611}
1612
1613static int cadence_nand_write_oob_raw(struct nand_chip *chip,
1614 int page)
1615{
1616 return cadence_nand_write_page_raw(chip, NULL, true, page);
1617}
1618
1619static int cadence_nand_read_page(struct nand_chip *chip,
1620 u8 *buf, int oob_required, int page)
1621{
1622 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1623 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1624 struct mtd_info *mtd = nand_to_mtd(chip);
1625 int status = 0;
1626 int ecc_err_count = 0;
1627
1628 status = cadence_nand_select_target(chip);
1629 if (status)
1630 return status;
1631
1632 cadence_nand_set_skip_bytes_conf(cdns_ctrl, cdns_chip->bbm_len,
1633 mtd->writesize
1634 + cdns_chip->bbm_offs, 1);
1635
1636 /*
1637 * If data buffer can be accessed by DMA and data_control feature
1638 * is supported then transfer data and oob directly.
1639 */
1640 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, mtd->writesize) &&
1641 cdns_ctrl->caps2.data_control_supp) {
1642 u8 *oob;
1643
1644 if (oob_required)
1645 oob = chip->oob_poi;
1646 else
1647 oob = cdns_ctrl->buf + mtd->writesize;
1648
1649 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREA_EXT);
1650 status = cadence_nand_cdma_transfer(cdns_ctrl,
1651 cdns_chip->cs[chip->cur_cs],
1652 page, buf, oob,
1653 mtd->writesize,
1654 cdns_chip->avail_oob_size,
1655 DMA_FROM_DEVICE, true);
1656 /* Otherwise use bounce buffer. */
1657 } else {
1658 cadence_nand_prepare_data_size(chip, TT_MAIN_OOB_AREAS);
1659 status = cadence_nand_cdma_transfer(cdns_ctrl,
1660 cdns_chip->cs[chip->cur_cs],
1661 page, cdns_ctrl->buf,
1662 NULL, mtd->writesize
1663 + cdns_chip->avail_oob_size,
1664 0, DMA_FROM_DEVICE, true);
1665
1666 memcpy(buf, cdns_ctrl->buf, mtd->writesize);
1667 if (oob_required)
1668 memcpy(chip->oob_poi,
1669 cdns_ctrl->buf + mtd->writesize,
1670 mtd->oobsize);
1671 }
1672
1673 switch (status) {
1674 case STAT_ECC_UNCORR:
1675 mtd->ecc_stats.failed++;
1676 ecc_err_count++;
1677 break;
1678 case STAT_ECC_CORR:
1679 ecc_err_count = FIELD_GET(CDMA_CS_MAXERR,
1680 cdns_ctrl->cdma_desc->status);
1681 mtd->ecc_stats.corrected += ecc_err_count;
1682 break;
1683 case STAT_ERASED:
1684 case STAT_OK:
1685 break;
1686 default:
1687 dev_err(cdns_ctrl->dev, "read page failed\n");
1688 return -EIO;
1689 }
1690
1691 if (oob_required)
1692 if (cadence_nand_read_bbm(chip, page, chip->oob_poi))
1693 return -EIO;
1694
1695 return ecc_err_count;
1696}
1697
1698/* Reads OOB data from the device. */
1699static int cadence_nand_read_oob(struct nand_chip *chip, int page)
1700{
1701 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1702
1703 return cadence_nand_read_page(chip, cdns_ctrl->buf, 1, page);
1704}
1705
1706static int cadence_nand_read_page_raw(struct nand_chip *chip,
1707 u8 *buf, int oob_required, int page)
1708{
1709 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
1710 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
1711 struct mtd_info *mtd = nand_to_mtd(chip);
1712 int oob_skip = cdns_chip->bbm_len;
1713 int writesize = mtd->writesize;
1714 int ecc_steps = chip->ecc.steps;
1715 int ecc_size = chip->ecc.size;
1716 int ecc_bytes = chip->ecc.bytes;
1717 void *tmp_buf = cdns_ctrl->buf;
1718 int i, pos, len;
1719 int status = 0;
1720
1721 status = cadence_nand_select_target(chip);
1722 if (status)
1723 return status;
1724
1725 cadence_nand_set_skip_bytes_conf(cdns_ctrl, 0, 0, 0);
1726
1727 cadence_nand_prepare_data_size(chip, TT_RAW_PAGE);
1728 status = cadence_nand_cdma_transfer(cdns_ctrl,
1729 cdns_chip->cs[chip->cur_cs],
1730 page, cdns_ctrl->buf, NULL,
1731 mtd->writesize
1732 + mtd->oobsize,
1733 0, DMA_FROM_DEVICE, false);
1734
1735 switch (status) {
1736 case STAT_ERASED:
1737 case STAT_OK:
1738 break;
1739 default:
1740 dev_err(cdns_ctrl->dev, "read raw page failed\n");
1741 return -EIO;
1742 }
1743
1744 /* Arrange the buffer for syndrome payload/ecc layout. */
1745 if (buf) {
1746 for (i = 0; i < ecc_steps; i++) {
1747 pos = i * (ecc_size + ecc_bytes);
1748 len = ecc_size;
1749
1750 if (pos >= writesize)
1751 pos += oob_skip;
1752 else if (pos + len > writesize)
1753 len = writesize - pos;
1754
1755 memcpy(buf, tmp_buf + pos, len);
1756 buf += len;
1757 if (len < ecc_size) {
1758 len = ecc_size - len;
1759 memcpy(buf, tmp_buf + writesize + oob_skip,
1760 len);
1761 buf += len;
1762 }
1763 }
1764 }
1765
1766 if (oob_required) {
1767 u8 *oob = chip->oob_poi;
1768 u32 oob_data_offset = (cdns_chip->sector_count - 1) *
1769 (cdns_chip->sector_size + chip->ecc.bytes)
1770 + cdns_chip->sector_size + oob_skip;
1771
1772 /* OOB free. */
1773 memcpy(oob, tmp_buf + oob_data_offset,
1774 cdns_chip->avail_oob_size);
1775
1776 /* BBM at the beginning of the OOB area. */
1777 memcpy(oob, tmp_buf + writesize, oob_skip);
1778
1779 oob += cdns_chip->avail_oob_size;
1780
1781 /* OOB ECC */
1782 for (i = 0; i < ecc_steps; i++) {
1783 pos = ecc_size + i * (ecc_size + ecc_bytes);
1784 len = ecc_bytes;
1785
1786 if (i == (ecc_steps - 1))
1787 pos += cdns_chip->avail_oob_size;
1788
1789 if (pos >= writesize)
1790 pos += oob_skip;
1791 else if (pos + len > writesize)
1792 len = writesize - pos;
1793
1794 memcpy(oob, tmp_buf + pos, len);
1795 oob += len;
1796 if (len < ecc_bytes) {
1797 len = ecc_bytes - len;
1798 memcpy(oob, tmp_buf + writesize + oob_skip,
1799 len);
1800 oob += len;
1801 }
1802 }
1803 }
1804
1805 return 0;
1806}
1807
1808static int cadence_nand_read_oob_raw(struct nand_chip *chip,
1809 int page)
1810{
1811 return cadence_nand_read_page_raw(chip, NULL, true, page);
1812}
1813
1814static void cadence_nand_slave_dma_transfer_finished(void *data)
1815{
1816 struct completion *finished = data;
1817
1818 complete(finished);
1819}
1820
1821static int cadence_nand_slave_dma_transfer(struct cdns_nand_ctrl *cdns_ctrl,
1822 void *buf,
1823 dma_addr_t dev_dma, size_t len,
1824 enum dma_data_direction dir)
1825{
1826 DECLARE_COMPLETION_ONSTACK(finished);
1827 struct dma_chan *chan;
1828 struct dma_device *dma_dev;
1829 dma_addr_t src_dma, dst_dma, buf_dma;
1830 struct dma_async_tx_descriptor *tx;
1831 dma_cookie_t cookie;
1832
1833 chan = cdns_ctrl->dmac;
1834 dma_dev = chan->device;
1835
1836 buf_dma = dma_map_single(dma_dev->dev, buf, len, dir);
1837 if (dma_mapping_error(dma_dev->dev, buf_dma)) {
1838 dev_err(cdns_ctrl->dev, "Failed to map DMA buffer\n");
1839 goto err;
1840 }
1841
1842 if (dir == DMA_FROM_DEVICE) {
1843 src_dma = cdns_ctrl->io.dma;
1844 dst_dma = buf_dma;
1845 } else {
1846 src_dma = buf_dma;
1847 dst_dma = cdns_ctrl->io.dma;
1848 }
1849
1850 tx = dmaengine_prep_dma_memcpy(cdns_ctrl->dmac, dst_dma, src_dma, len,
1851 DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
1852 if (!tx) {
1853 dev_err(cdns_ctrl->dev, "Failed to prepare DMA memcpy\n");
1854 goto err_unmap;
1855 }
1856
1857 tx->callback = cadence_nand_slave_dma_transfer_finished;
1858 tx->callback_param = &finished;
1859
1860 cookie = dmaengine_submit(tx);
1861 if (dma_submit_error(cookie)) {
1862 dev_err(cdns_ctrl->dev, "Failed to do DMA tx_submit\n");
1863 goto err_unmap;
1864 }
1865
1866 dma_async_issue_pending(cdns_ctrl->dmac);
1867 wait_for_completion(&finished);
1868
1869 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1870
1871 return 0;
1872
1873err_unmap:
1874 dma_unmap_single(cdns_ctrl->dev, buf_dma, len, dir);
1875
1876err:
1877 dev_dbg(cdns_ctrl->dev, "Fall back to CPU I/O\n");
1878
1879 return -EIO;
1880}
1881
1882static int cadence_nand_read_buf(struct cdns_nand_ctrl *cdns_ctrl,
1883 u8 *buf, int len)
1884{
1885 u8 thread_nr = 0;
1886 u32 sdma_size;
1887 int status;
1888
1889 /* Wait until slave DMA interface is ready to data transfer. */
1890 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1891 if (status)
1892 return status;
1893
1894 if (!cdns_ctrl->caps1->has_dma) {
1895 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1896
1897 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1898
1899 /* read alingment data */
1900 if (data_dma_width == 4)
1901 ioread32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1902#ifdef CONFIG_64BIT
1903 else
1904 readsq(cdns_ctrl->io.virt, buf, len_in_words);
1905#endif
1906
1907 if (sdma_size > len) {
1908 int read_bytes = (data_dma_width == 4) ?
1909 len_in_words << 2 : len_in_words << 3;
1910
1911 /* read rest data from slave DMA interface if any */
1912 if (data_dma_width == 4)
1913 ioread32_rep(cdns_ctrl->io.virt,
1914 cdns_ctrl->buf,
1915 sdma_size / 4 - len_in_words);
1916#ifdef CONFIG_64BIT
1917 else
1918 readsq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1919 sdma_size / 8 - len_in_words);
1920#endif
1921
1922 /* copy rest of data */
1923 memcpy(buf + read_bytes, cdns_ctrl->buf,
1924 len - read_bytes);
1925 }
1926 return 0;
1927 }
1928
1929 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
1930 status = cadence_nand_slave_dma_transfer(cdns_ctrl, buf,
1931 cdns_ctrl->io.dma,
1932 len, DMA_FROM_DEVICE);
1933 if (status == 0)
1934 return 0;
1935
1936 dev_warn(cdns_ctrl->dev,
1937 "Slave DMA transfer failed. Try again using bounce buffer.");
1938 }
1939
1940 /* If DMA transfer is not possible or failed then use bounce buffer. */
1941 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
1942 cdns_ctrl->io.dma,
1943 sdma_size, DMA_FROM_DEVICE);
1944
1945 if (status) {
1946 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
1947 return status;
1948 }
1949
1950 memcpy(buf, cdns_ctrl->buf, len);
1951
1952 return 0;
1953}
1954
1955static int cadence_nand_write_buf(struct cdns_nand_ctrl *cdns_ctrl,
1956 const u8 *buf, int len)
1957{
1958 u8 thread_nr = 0;
1959 u32 sdma_size;
1960 int status;
1961
1962 /* Wait until slave DMA interface is ready to data transfer. */
1963 status = cadence_nand_wait_on_sdma(cdns_ctrl, &thread_nr, &sdma_size);
1964 if (status)
1965 return status;
1966
1967 if (!cdns_ctrl->caps1->has_dma) {
1968 u8 data_dma_width = cdns_ctrl->caps2.data_dma_width;
1969
1970 int len_in_words = (data_dma_width == 4) ? len >> 2 : len >> 3;
1971
1972 if (data_dma_width == 4)
1973 iowrite32_rep(cdns_ctrl->io.virt, buf, len_in_words);
1974#ifdef CONFIG_64BIT
1975 else
1976 writesq(cdns_ctrl->io.virt, buf, len_in_words);
1977#endif
1978
1979 if (sdma_size > len) {
1980 int written_bytes = (data_dma_width == 4) ?
1981 len_in_words << 2 : len_in_words << 3;
1982
1983 /* copy rest of data */
1984 memcpy(cdns_ctrl->buf, buf + written_bytes,
1985 len - written_bytes);
1986
1987 /* write all expected by nand controller data */
1988 if (data_dma_width == 4)
1989 iowrite32_rep(cdns_ctrl->io.virt,
1990 cdns_ctrl->buf,
1991 sdma_size / 4 - len_in_words);
1992#ifdef CONFIG_64BIT
1993 else
1994 writesq(cdns_ctrl->io.virt, cdns_ctrl->buf,
1995 sdma_size / 8 - len_in_words);
1996#endif
1997 }
1998
1999 return 0;
2000 }
2001
2002 if (cadence_nand_dma_buf_ok(cdns_ctrl, buf, len)) {
2003 status = cadence_nand_slave_dma_transfer(cdns_ctrl, (void *)buf,
2004 cdns_ctrl->io.dma,
2005 len, DMA_TO_DEVICE);
2006 if (status == 0)
2007 return 0;
2008
2009 dev_warn(cdns_ctrl->dev,
2010 "Slave DMA transfer failed. Try again using bounce buffer.");
2011 }
2012
2013 /* If DMA transfer is not possible or failed then use bounce buffer. */
2014 memcpy(cdns_ctrl->buf, buf, len);
2015
2016 status = cadence_nand_slave_dma_transfer(cdns_ctrl, cdns_ctrl->buf,
2017 cdns_ctrl->io.dma,
2018 sdma_size, DMA_TO_DEVICE);
2019
2020 if (status)
2021 dev_err(cdns_ctrl->dev, "Slave DMA transfer failed");
2022
2023 return status;
2024}
2025
2026static int cadence_nand_force_byte_access(struct nand_chip *chip,
2027 bool force_8bit)
2028{
2029 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2030
2031 /*
2032 * Callers of this function do not verify if the NAND is using a 16-bit
2033 * an 8-bit bus for normal operations, so we need to take care of that
2034 * here by leaving the configuration unchanged if the NAND does not have
2035 * the NAND_BUSWIDTH_16 flag set.
2036 */
2037 if (!(chip->options & NAND_BUSWIDTH_16))
2038 return 0;
2039
2040 return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit);
2041}
2042
2043static int cadence_nand_cmd_opcode(struct nand_chip *chip,
2044 const struct nand_subop *subop)
2045{
2046 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2047 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2048 const struct nand_op_instr *instr;
2049 unsigned int op_id = 0;
2050 u64 mini_ctrl_cmd = 0;
2051 int ret;
2052
2053 instr = &subop->instrs[op_id];
2054
2055 if (instr->delay_ns > 0)
2056 mini_ctrl_cmd |= GCMD_LAY_TWB;
2057
2058 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2059 GCMD_LAY_INSTR_CMD);
2060 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_CMD,
2061 instr->ctx.cmd.opcode);
2062
2063 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2064 cdns_chip->cs[chip->cur_cs],
2065 mini_ctrl_cmd);
2066 if (ret)
2067 dev_err(cdns_ctrl->dev, "send cmd %x failed\n",
2068 instr->ctx.cmd.opcode);
2069
2070 return ret;
2071}
2072
2073static int cadence_nand_cmd_address(struct nand_chip *chip,
2074 const struct nand_subop *subop)
2075{
2076 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2077 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2078 const struct nand_op_instr *instr;
2079 unsigned int op_id = 0;
2080 u64 mini_ctrl_cmd = 0;
2081 unsigned int offset, naddrs;
2082 u64 address = 0;
2083 const u8 *addrs;
2084 int ret;
2085 int i;
2086
2087 instr = &subop->instrs[op_id];
2088
2089 if (instr->delay_ns > 0)
2090 mini_ctrl_cmd |= GCMD_LAY_TWB;
2091
2092 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2093 GCMD_LAY_INSTR_ADDR);
2094
2095 offset = nand_subop_get_addr_start_off(subop, op_id);
2096 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
2097 addrs = &instr->ctx.addr.addrs[offset];
2098
2099 for (i = 0; i < naddrs; i++)
2100 address |= (u64)addrs[i] << (8 * i);
2101
2102 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR,
2103 address);
2104 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INPUT_ADDR_SIZE,
2105 naddrs - 1);
2106
2107 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2108 cdns_chip->cs[chip->cur_cs],
2109 mini_ctrl_cmd);
2110 if (ret)
2111 dev_err(cdns_ctrl->dev, "send address %llx failed\n", address);
2112
2113 return ret;
2114}
2115
2116static int cadence_nand_cmd_erase(struct nand_chip *chip,
2117 const struct nand_subop *subop)
2118{
2119 unsigned int op_id;
2120
2121 if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_ERASE1) {
2122 int i;
2123 const struct nand_op_instr *instr = NULL;
2124 unsigned int offset, naddrs;
2125 const u8 *addrs;
2126 u32 page = 0;
2127
2128 instr = &subop->instrs[1];
2129 offset = nand_subop_get_addr_start_off(subop, 1);
2130 naddrs = nand_subop_get_num_addr_cyc(subop, 1);
2131 addrs = &instr->ctx.addr.addrs[offset];
2132
2133 for (i = 0; i < naddrs; i++)
2134 page |= (u32)addrs[i] << (8 * i);
2135
2136 return cadence_nand_erase(chip, page);
2137 }
2138
2139 /*
2140 * If it is not an erase operation then handle operation
2141 * by calling exec_op function.
2142 */
2143 for (op_id = 0; op_id < subop->ninstrs; op_id++) {
2144 int ret;
2145 const struct nand_operation nand_op = {
2146 .cs = chip->cur_cs,
2147 .instrs = &subop->instrs[op_id],
2148 .ninstrs = 1};
2149 ret = chip->controller->ops->exec_op(chip, &nand_op, false);
2150 if (ret)
2151 return ret;
2152 }
2153
2154 return 0;
2155}
2156
2157static int cadence_nand_cmd_data(struct nand_chip *chip,
2158 const struct nand_subop *subop)
2159{
2160 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2161 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2162 const struct nand_op_instr *instr;
2163 unsigned int offset, op_id = 0;
2164 u64 mini_ctrl_cmd = 0;
2165 int len = 0;
2166 int ret;
2167
2168 instr = &subop->instrs[op_id];
2169
2170 if (instr->delay_ns > 0)
2171 mini_ctrl_cmd |= GCMD_LAY_TWB;
2172
2173 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAY_INSTR,
2174 GCMD_LAY_INSTR_DATA);
2175
2176 if (instr->type == NAND_OP_DATA_OUT_INSTR)
2177 mini_ctrl_cmd |= FIELD_PREP(GCMD_DIR,
2178 GCMD_DIR_WRITE);
2179
2180 len = nand_subop_get_data_len(subop, op_id);
2181 offset = nand_subop_get_data_start_off(subop, op_id);
2182 mini_ctrl_cmd |= FIELD_PREP(GCMD_SECT_CNT, 1);
2183 mini_ctrl_cmd |= FIELD_PREP(GCMD_LAST_SIZE, len);
2184 if (instr->ctx.data.force_8bit) {
2185 ret = cadence_nand_force_byte_access(chip, true);
2186 if (ret) {
2187 dev_err(cdns_ctrl->dev,
2188 "cannot change byte access generic data cmd failed\n");
2189 return ret;
2190 }
2191 }
2192
2193 ret = cadence_nand_generic_cmd_send(cdns_ctrl,
2194 cdns_chip->cs[chip->cur_cs],
2195 mini_ctrl_cmd);
2196 if (ret) {
2197 dev_err(cdns_ctrl->dev, "send generic data cmd failed\n");
2198 return ret;
2199 }
2200
2201 if (instr->type == NAND_OP_DATA_IN_INSTR) {
2202 void *buf = instr->ctx.data.buf.in + offset;
2203
2204 ret = cadence_nand_read_buf(cdns_ctrl, buf, len);
2205 } else {
2206 const void *buf = instr->ctx.data.buf.out + offset;
2207
2208 ret = cadence_nand_write_buf(cdns_ctrl, buf, len);
2209 }
2210
2211 if (ret) {
2212 dev_err(cdns_ctrl->dev, "data transfer failed for generic command\n");
2213 return ret;
2214 }
2215
2216 if (instr->ctx.data.force_8bit) {
2217 ret = cadence_nand_force_byte_access(chip, false);
2218 if (ret) {
2219 dev_err(cdns_ctrl->dev,
2220 "cannot change byte access generic data cmd failed\n");
2221 }
2222 }
2223
2224 return ret;
2225}
2226
2227static int cadence_nand_cmd_waitrdy(struct nand_chip *chip,
2228 const struct nand_subop *subop)
2229{
2230 int status;
2231 unsigned int op_id = 0;
2232 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2233 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2234 const struct nand_op_instr *instr = &subop->instrs[op_id];
2235 u32 timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
2236
2237 status = cadence_nand_wait_for_value(cdns_ctrl, RBN_SETINGS,
2238 timeout_us,
2239 BIT(cdns_chip->cs[chip->cur_cs]),
2240 false);
2241 return status;
2242}
2243
2244static const struct nand_op_parser cadence_nand_op_parser = NAND_OP_PARSER(
2245 NAND_OP_PARSER_PATTERN(
2246 cadence_nand_cmd_erase,
2247 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2248 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ERASE_ADDRESS_CYC),
2249 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2250 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2251 NAND_OP_PARSER_PATTERN(
2252 cadence_nand_cmd_opcode,
2253 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2254 NAND_OP_PARSER_PATTERN(
2255 cadence_nand_cmd_address,
2256 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC)),
2257 NAND_OP_PARSER_PATTERN(
2258 cadence_nand_cmd_data,
2259 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_DATA_SIZE)),
2260 NAND_OP_PARSER_PATTERN(
2261 cadence_nand_cmd_data,
2262 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_DATA_SIZE)),
2263 NAND_OP_PARSER_PATTERN(
2264 cadence_nand_cmd_waitrdy,
2265 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false))
2266 );
2267
2268static int cadence_nand_exec_op(struct nand_chip *chip,
2269 const struct nand_operation *op,
2270 bool check_only)
2271{
2272 if (!check_only) {
2273 int status = cadence_nand_select_target(chip);
2274
2275 if (status)
2276 return status;
2277 }
2278
2279 return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
2280 check_only);
2281}
2282
2283static int cadence_nand_ooblayout_free(struct mtd_info *mtd, int section,
2284 struct mtd_oob_region *oobregion)
2285{
2286 struct nand_chip *chip = mtd_to_nand(mtd);
2287 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2288
2289 if (section)
2290 return -ERANGE;
2291
2292 oobregion->offset = cdns_chip->bbm_len;
2293 oobregion->length = cdns_chip->avail_oob_size
2294 - cdns_chip->bbm_len;
2295
2296 return 0;
2297}
2298
2299static int cadence_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2300 struct mtd_oob_region *oobregion)
2301{
2302 struct nand_chip *chip = mtd_to_nand(mtd);
2303 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2304
2305 if (section)
2306 return -ERANGE;
2307
2308 oobregion->offset = cdns_chip->avail_oob_size;
2309 oobregion->length = chip->ecc.total;
2310
2311 return 0;
2312}
2313
2314static const struct mtd_ooblayout_ops cadence_nand_ooblayout_ops = {
2315 .free = cadence_nand_ooblayout_free,
2316 .ecc = cadence_nand_ooblayout_ecc,
2317};
2318
2319static int calc_cycl(u32 timing, u32 clock)
2320{
2321 if (timing == 0 || clock == 0)
2322 return 0;
2323
2324 if ((timing % clock) > 0)
2325 return timing / clock;
2326 else
2327 return timing / clock - 1;
2328}
2329
2330/* Calculate max data valid window. */
2331static inline u32 calc_tdvw_max(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2332 u32 board_delay_skew_min, u32 ext_mode)
2333{
2334 if (ext_mode == 0)
2335 clk_period /= 2;
2336
2337 return (trp_cnt + 1) * clk_period + trhoh_min +
2338 board_delay_skew_min;
2339}
2340
2341/* Calculate data valid window. */
2342static inline u32 calc_tdvw(u32 trp_cnt, u32 clk_period, u32 trhoh_min,
2343 u32 trea_max, u32 ext_mode)
2344{
2345 if (ext_mode == 0)
2346 clk_period /= 2;
2347
2348 return (trp_cnt + 1) * clk_period + trhoh_min - trea_max;
2349}
2350
2351static int
2352cadence_nand_setup_interface(struct nand_chip *chip, int chipnr,
2353 const struct nand_interface_config *conf)
2354{
2355 const struct nand_sdr_timings *sdr;
2356 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2357 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2358 struct cadence_nand_timings *t = &cdns_chip->timings;
2359 u32 reg;
2360 u32 board_delay = cdns_ctrl->board_delay;
2361 u32 clk_period = DIV_ROUND_DOWN_ULL(1000000000000ULL,
2362 cdns_ctrl->nf_clk_rate);
2363 u32 tceh_cnt, tcs_cnt, tadl_cnt, tccs_cnt;
2364 u32 tfeat_cnt, trhz_cnt, tvdly_cnt;
2365 u32 trhw_cnt, twb_cnt, twh_cnt = 0, twhr_cnt;
2366 u32 twp_cnt = 0, trp_cnt = 0, trh_cnt = 0;
2367 u32 if_skew = cdns_ctrl->caps1->if_skew;
2368 u32 board_delay_skew_min = board_delay - if_skew;
2369 u32 board_delay_skew_max = board_delay + if_skew;
2370 u32 dqs_sampl_res, phony_dqs_mod;
2371 u32 tdvw, tdvw_min, tdvw_max;
2372 u32 ext_rd_mode, ext_wr_mode;
2373 u32 dll_phy_dqs_timing = 0, phony_dqs_timing = 0, rd_del_sel = 0;
2374 u32 sampling_point;
2375
2376 sdr = nand_get_sdr_timings(conf);
2377 if (IS_ERR(sdr))
2378 return PTR_ERR(sdr);
2379
2380 memset(t, 0, sizeof(*t));
2381 /* Sampling point calculation. */
2382
2383 if (cdns_ctrl->caps2.is_phy_type_dll)
2384 phony_dqs_mod = 2;
2385 else
2386 phony_dqs_mod = 1;
2387
2388 dqs_sampl_res = clk_period / phony_dqs_mod;
2389
2390 tdvw_min = sdr->tREA_max + board_delay_skew_max;
2391 /*
2392 * The idea of those calculation is to get the optimum value
2393 * for tRP and tRH timings. If it is NOT possible to sample data
2394 * with optimal tRP/tRH settings, the parameters will be extended.
2395 * If clk_period is 50ns (the lowest value) this condition is met
2396 * for SDR timing modes 1, 2, 3, 4 and 5.
2397 * If clk_period is 20ns the condition is met only for SDR timing
2398 * mode 5.
2399 */
2400 if (sdr->tRC_min <= clk_period &&
2401 sdr->tRP_min <= (clk_period / 2) &&
2402 sdr->tREH_min <= (clk_period / 2)) {
2403 /* Performance mode. */
2404 ext_rd_mode = 0;
2405 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2406 sdr->tREA_max, ext_rd_mode);
2407 tdvw_max = calc_tdvw_max(trp_cnt, clk_period, sdr->tRHOH_min,
2408 board_delay_skew_min,
2409 ext_rd_mode);
2410 /*
2411 * Check if data valid window and sampling point can be found
2412 * and is not on the edge (ie. we have hold margin).
2413 * If not extend the tRP timings.
2414 */
2415 if (tdvw > 0) {
2416 if (tdvw_max <= tdvw_min ||
2417 (tdvw_max % dqs_sampl_res) == 0) {
2418 /*
2419 * No valid sampling point so the RE pulse need
2420 * to be widen widening by half clock cycle.
2421 */
2422 ext_rd_mode = 1;
2423 }
2424 } else {
2425 /*
2426 * There is no valid window
2427 * to be able to sample data the tRP need to be widen.
2428 * Very safe calculations are performed here.
2429 */
2430 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2431 + dqs_sampl_res) / clk_period;
2432 ext_rd_mode = 1;
2433 }
2434
2435 } else {
2436 /* Extended read mode. */
2437 u32 trh;
2438
2439 ext_rd_mode = 1;
2440 trp_cnt = calc_cycl(sdr->tRP_min, clk_period);
2441 trh = sdr->tRC_min - ((trp_cnt + 1) * clk_period);
2442 if (sdr->tREH_min >= trh)
2443 trh_cnt = calc_cycl(sdr->tREH_min, clk_period);
2444 else
2445 trh_cnt = calc_cycl(trh, clk_period);
2446
2447 tdvw = calc_tdvw(trp_cnt, clk_period, sdr->tRHOH_min,
2448 sdr->tREA_max, ext_rd_mode);
2449 /*
2450 * Check if data valid window and sampling point can be found
2451 * or if it is at the edge check if previous is valid
2452 * - if not extend the tRP timings.
2453 */
2454 if (tdvw > 0) {
2455 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2456 sdr->tRHOH_min,
2457 board_delay_skew_min,
2458 ext_rd_mode);
2459
2460 if ((((tdvw_max / dqs_sampl_res)
2461 * dqs_sampl_res) <= tdvw_min) ||
2462 (((tdvw_max % dqs_sampl_res) == 0) &&
2463 (((tdvw_max / dqs_sampl_res - 1)
2464 * dqs_sampl_res) <= tdvw_min))) {
2465 /*
2466 * Data valid window width is lower than
2467 * sampling resolution and do not hit any
2468 * sampling point to be sure the sampling point
2469 * will be found the RE low pulse width will be
2470 * extended by one clock cycle.
2471 */
2472 trp_cnt = trp_cnt + 1;
2473 }
2474 } else {
2475 /*
2476 * There is no valid window to be able to sample data.
2477 * The tRP need to be widen.
2478 * Very safe calculations are performed here.
2479 */
2480 trp_cnt = (sdr->tREA_max + board_delay_skew_max
2481 + dqs_sampl_res) / clk_period;
2482 }
2483 }
2484
2485 tdvw_max = calc_tdvw_max(trp_cnt, clk_period,
2486 sdr->tRHOH_min,
2487 board_delay_skew_min, ext_rd_mode);
2488
2489 if (sdr->tWC_min <= clk_period &&
2490 (sdr->tWP_min + if_skew) <= (clk_period / 2) &&
2491 (sdr->tWH_min + if_skew) <= (clk_period / 2)) {
2492 ext_wr_mode = 0;
2493 } else {
2494 u32 twh;
2495
2496 ext_wr_mode = 1;
2497 twp_cnt = calc_cycl(sdr->tWP_min + if_skew, clk_period);
2498 if ((twp_cnt + 1) * clk_period < (sdr->tALS_min + if_skew))
2499 twp_cnt = calc_cycl(sdr->tALS_min + if_skew,
2500 clk_period);
2501
2502 twh = (sdr->tWC_min - (twp_cnt + 1) * clk_period);
2503 if (sdr->tWH_min >= twh)
2504 twh = sdr->tWH_min;
2505
2506 twh_cnt = calc_cycl(twh + if_skew, clk_period);
2507 }
2508
2509 reg = FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRH, trh_cnt);
2510 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TRP, trp_cnt);
2511 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWH, twh_cnt);
2512 reg |= FIELD_PREP(ASYNC_TOGGLE_TIMINGS_TWP, twp_cnt);
2513 t->async_toggle_timings = reg;
2514 dev_dbg(cdns_ctrl->dev, "ASYNC_TOGGLE_TIMINGS_SDR\t%x\n", reg);
2515
2516 tadl_cnt = calc_cycl((sdr->tADL_min + if_skew), clk_period);
2517 tccs_cnt = calc_cycl((sdr->tCCS_min + if_skew), clk_period);
2518 twhr_cnt = calc_cycl((sdr->tWHR_min + if_skew), clk_period);
2519 trhw_cnt = calc_cycl((sdr->tRHW_min + if_skew), clk_period);
2520 reg = FIELD_PREP(TIMINGS0_TADL, tadl_cnt);
2521
2522 /*
2523 * If timing exceeds delay field in timing register
2524 * then use maximum value.
2525 */
2526 if (FIELD_FIT(TIMINGS0_TCCS, tccs_cnt))
2527 reg |= FIELD_PREP(TIMINGS0_TCCS, tccs_cnt);
2528 else
2529 reg |= TIMINGS0_TCCS;
2530
2531 reg |= FIELD_PREP(TIMINGS0_TWHR, twhr_cnt);
2532 reg |= FIELD_PREP(TIMINGS0_TRHW, trhw_cnt);
2533 t->timings0 = reg;
2534 dev_dbg(cdns_ctrl->dev, "TIMINGS0_SDR\t%x\n", reg);
2535
2536 /* The following is related to single signal so skew is not needed. */
2537 trhz_cnt = calc_cycl(sdr->tRHZ_max, clk_period);
2538 trhz_cnt = trhz_cnt + 1;
2539 twb_cnt = calc_cycl((sdr->tWB_max + board_delay), clk_period);
2540 /*
2541 * Because of the two stage syncflop the value must be increased by 3
2542 * first value is related with sync, second value is related
2543 * with output if delay.
2544 */
2545 twb_cnt = twb_cnt + 3 + 5;
2546 /*
2547 * The following is related to the we edge of the random data input
2548 * sequence so skew is not needed.
2549 */
2550 tvdly_cnt = calc_cycl(500000 + if_skew, clk_period);
2551 reg = FIELD_PREP(TIMINGS1_TRHZ, trhz_cnt);
2552 reg |= FIELD_PREP(TIMINGS1_TWB, twb_cnt);
2553 reg |= FIELD_PREP(TIMINGS1_TVDLY, tvdly_cnt);
2554 t->timings1 = reg;
2555 dev_dbg(cdns_ctrl->dev, "TIMINGS1_SDR\t%x\n", reg);
2556
2557 tfeat_cnt = calc_cycl(sdr->tFEAT_max, clk_period);
2558 if (tfeat_cnt < twb_cnt)
2559 tfeat_cnt = twb_cnt;
2560
2561 tceh_cnt = calc_cycl(sdr->tCEH_min, clk_period);
2562 tcs_cnt = calc_cycl((sdr->tCS_min + if_skew), clk_period);
2563
2564 reg = FIELD_PREP(TIMINGS2_TFEAT, tfeat_cnt);
2565 reg |= FIELD_PREP(TIMINGS2_CS_HOLD_TIME, tceh_cnt);
2566 reg |= FIELD_PREP(TIMINGS2_CS_SETUP_TIME, tcs_cnt);
2567 t->timings2 = reg;
2568 dev_dbg(cdns_ctrl->dev, "TIMINGS2_SDR\t%x\n", reg);
2569
2570 if (cdns_ctrl->caps2.is_phy_type_dll) {
2571 reg = DLL_PHY_CTRL_DLL_RST_N;
2572 if (ext_wr_mode)
2573 reg |= DLL_PHY_CTRL_EXTENDED_WR_MODE;
2574 if (ext_rd_mode)
2575 reg |= DLL_PHY_CTRL_EXTENDED_RD_MODE;
2576
2577 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_HIGH_WAIT_CNT, 7);
2578 reg |= FIELD_PREP(DLL_PHY_CTRL_RS_IDLE_CNT, 7);
2579 t->dll_phy_ctrl = reg;
2580 dev_dbg(cdns_ctrl->dev, "DLL_PHY_CTRL_SDR\t%x\n", reg);
2581 }
2582
2583 /* Sampling point calculation. */
2584 if ((tdvw_max % dqs_sampl_res) > 0)
2585 sampling_point = tdvw_max / dqs_sampl_res;
2586 else
2587 sampling_point = (tdvw_max / dqs_sampl_res - 1);
2588
2589 if (sampling_point * dqs_sampl_res > tdvw_min) {
2590 dll_phy_dqs_timing =
2591 FIELD_PREP(PHY_DQS_TIMING_DQS_SEL_OE_END, 4);
2592 dll_phy_dqs_timing |= PHY_DQS_TIMING_USE_PHONY_DQS;
2593 phony_dqs_timing = sampling_point / phony_dqs_mod;
2594
2595 if ((sampling_point % 2) > 0) {
2596 dll_phy_dqs_timing |= PHY_DQS_TIMING_PHONY_DQS_SEL;
2597 if ((tdvw_max % dqs_sampl_res) == 0)
2598 /*
2599 * Calculation for sampling point at the edge
2600 * of data and being odd number.
2601 */
2602 phony_dqs_timing = (tdvw_max / dqs_sampl_res)
2603 / phony_dqs_mod - 1;
2604
2605 if (!cdns_ctrl->caps2.is_phy_type_dll)
2606 phony_dqs_timing--;
2607
2608 } else {
2609 phony_dqs_timing--;
2610 }
2611 rd_del_sel = phony_dqs_timing + 3;
2612 } else {
2613 dev_warn(cdns_ctrl->dev,
2614 "ERROR : cannot find valid sampling point\n");
2615 }
2616
2617 reg = FIELD_PREP(PHY_CTRL_PHONY_DQS, phony_dqs_timing);
2618 if (cdns_ctrl->caps2.is_phy_type_dll)
2619 reg |= PHY_CTRL_SDR_DQS;
2620 t->phy_ctrl = reg;
2621 dev_dbg(cdns_ctrl->dev, "PHY_CTRL_REG_SDR\t%x\n", reg);
2622
2623 if (cdns_ctrl->caps2.is_phy_type_dll) {
2624 dev_dbg(cdns_ctrl->dev, "PHY_TSEL_REG_SDR\t%x\n", 0);
2625 dev_dbg(cdns_ctrl->dev, "PHY_DQ_TIMING_REG_SDR\t%x\n", 2);
2626 dev_dbg(cdns_ctrl->dev, "PHY_DQS_TIMING_REG_SDR\t%x\n",
2627 dll_phy_dqs_timing);
2628 t->phy_dqs_timing = dll_phy_dqs_timing;
2629
2630 reg = FIELD_PREP(PHY_GATE_LPBK_CTRL_RDS, rd_del_sel);
2631 dev_dbg(cdns_ctrl->dev, "PHY_GATE_LPBK_CTRL_REG_SDR\t%x\n",
2632 reg);
2633 t->phy_gate_lpbk_ctrl = reg;
2634
2635 dev_dbg(cdns_ctrl->dev, "PHY_DLL_MASTER_CTRL_REG_SDR\t%lx\n",
2636 PHY_DLL_MASTER_CTRL_BYPASS_MODE);
2637 dev_dbg(cdns_ctrl->dev, "PHY_DLL_SLAVE_CTRL_REG_SDR\t%x\n", 0);
2638 }
2639
2640 return 0;
2641}
2642
2643static int cadence_nand_attach_chip(struct nand_chip *chip)
2644{
2645 struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
2646 struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
2647 u32 ecc_size;
2648 struct mtd_info *mtd = nand_to_mtd(chip);
2649 int ret;
2650
2651 if (chip->options & NAND_BUSWIDTH_16) {
2652 ret = cadence_nand_set_access_width16(cdns_ctrl, true);
2653 if (ret)
2654 return ret;
2655 }
2656
2657 chip->bbt_options |= NAND_BBT_USE_FLASH;
2658 chip->bbt_options |= NAND_BBT_NO_OOB;
2659 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2660
2661 chip->options |= NAND_NO_SUBPAGE_WRITE;
2662
2663 cdns_chip->bbm_offs = chip->badblockpos;
2664 cdns_chip->bbm_offs &= ~0x01;
2665 /* this value should be even number */
2666 cdns_chip->bbm_len = 2;
2667
2668 ret = nand_ecc_choose_conf(chip,
2669 &cdns_ctrl->ecc_caps,
2670 mtd->oobsize - cdns_chip->bbm_len);
2671 if (ret) {
2672 dev_err(cdns_ctrl->dev, "ECC configuration failed\n");
2673 return ret;
2674 }
2675
2676 dev_dbg(cdns_ctrl->dev,
2677 "chosen ECC settings: step=%d, strength=%d, bytes=%d\n",
2678 chip->ecc.size, chip->ecc.strength, chip->ecc.bytes);
2679
2680 /* Error correction configuration. */
2681 cdns_chip->sector_size = chip->ecc.size;
2682 cdns_chip->sector_count = mtd->writesize / cdns_chip->sector_size;
2683 ecc_size = cdns_chip->sector_count * chip->ecc.bytes;
2684
2685 cdns_chip->avail_oob_size = mtd->oobsize - ecc_size;
2686
2687 if (cdns_chip->avail_oob_size > cdns_ctrl->bch_metadata_size)
2688 cdns_chip->avail_oob_size = cdns_ctrl->bch_metadata_size;
2689
2690 if ((cdns_chip->avail_oob_size + cdns_chip->bbm_len + ecc_size)
2691 > mtd->oobsize)
2692 cdns_chip->avail_oob_size -= 4;
2693
2694 ret = cadence_nand_get_ecc_strength_idx(cdns_ctrl, chip->ecc.strength);
2695 if (ret < 0)
2696 return -EINVAL;
2697
2698 cdns_chip->corr_str_idx = (u8)ret;
2699
2700 if (cadence_nand_wait_for_value(cdns_ctrl, CTRL_STATUS,
2701 1000000,
2702 CTRL_STATUS_CTRL_BUSY, true))
2703 return -ETIMEDOUT;
2704
2705 cadence_nand_set_ecc_strength(cdns_ctrl,
2706 cdns_chip->corr_str_idx);
2707
2708 cadence_nand_set_erase_detection(cdns_ctrl, true,
2709 chip->ecc.strength);
2710
2711 /* Override the default read operations. */
2712 chip->ecc.read_page = cadence_nand_read_page;
2713 chip->ecc.read_page_raw = cadence_nand_read_page_raw;
2714 chip->ecc.write_page = cadence_nand_write_page;
2715 chip->ecc.write_page_raw = cadence_nand_write_page_raw;
2716 chip->ecc.read_oob = cadence_nand_read_oob;
2717 chip->ecc.write_oob = cadence_nand_write_oob;
2718 chip->ecc.read_oob_raw = cadence_nand_read_oob_raw;
2719 chip->ecc.write_oob_raw = cadence_nand_write_oob_raw;
2720
2721 if ((mtd->writesize + mtd->oobsize) > cdns_ctrl->buf_size)
2722 cdns_ctrl->buf_size = mtd->writesize + mtd->oobsize;
2723
2724 /* Is 32-bit DMA supported? */
2725 ret = dma_set_mask(cdns_ctrl->dev, DMA_BIT_MASK(32));
2726 if (ret) {
2727 dev_err(cdns_ctrl->dev, "no usable DMA configuration\n");
2728 return ret;
2729 }
2730
2731 mtd_set_ooblayout(mtd, &cadence_nand_ooblayout_ops);
2732
2733 return 0;
2734}
2735
2736static const struct nand_controller_ops cadence_nand_controller_ops = {
2737 .attach_chip = cadence_nand_attach_chip,
2738 .exec_op = cadence_nand_exec_op,
2739 .setup_interface = cadence_nand_setup_interface,
2740};
2741
2742static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
2743 struct device_node *np)
2744{
2745 struct cdns_nand_chip *cdns_chip;
2746 struct mtd_info *mtd;
2747 struct nand_chip *chip;
2748 int nsels, ret, i;
2749 u32 cs;
2750
2751 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2752 if (nsels <= 0) {
2753 dev_err(cdns_ctrl->dev, "missing/invalid reg property\n");
2754 return -EINVAL;
2755 }
2756
2757 /* Allocate the nand chip structure. */
2758 cdns_chip = devm_kzalloc(cdns_ctrl->dev, sizeof(*cdns_chip) +
2759 (nsels * sizeof(u8)),
2760 GFP_KERNEL);
2761 if (!cdns_chip) {
2762 dev_err(cdns_ctrl->dev, "could not allocate chip structure\n");
2763 return -ENOMEM;
2764 }
2765
2766 cdns_chip->nsels = nsels;
2767
2768 for (i = 0; i < nsels; i++) {
2769 /* Retrieve CS id. */
2770 ret = of_property_read_u32_index(np, "reg", i, &cs);
2771 if (ret) {
2772 dev_err(cdns_ctrl->dev,
2773 "could not retrieve reg property: %d\n",
2774 ret);
2775 return ret;
2776 }
2777
2778 if (cs >= cdns_ctrl->caps2.max_banks) {
2779 dev_err(cdns_ctrl->dev,
2780 "invalid reg value: %u (max CS = %d)\n",
2781 cs, cdns_ctrl->caps2.max_banks);
2782 return -EINVAL;
2783 }
2784
2785 if (test_and_set_bit(cs, &cdns_ctrl->assigned_cs)) {
2786 dev_err(cdns_ctrl->dev,
2787 "CS %d already assigned\n", cs);
2788 return -EINVAL;
2789 }
2790
2791 cdns_chip->cs[i] = cs;
2792 }
2793
2794 chip = &cdns_chip->chip;
2795 chip->controller = &cdns_ctrl->controller;
2796 nand_set_flash_node(chip, np);
2797
2798 mtd = nand_to_mtd(chip);
2799 mtd->dev.parent = cdns_ctrl->dev;
2800
2801 /*
2802 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2803 * in the DT node, this entry will be overwritten in nand_scan_ident().
2804 */
2805 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2806
2807 ret = nand_scan(chip, cdns_chip->nsels);
2808 if (ret) {
2809 dev_err(cdns_ctrl->dev, "could not scan the nand chip\n");
2810 return ret;
2811 }
2812
2813 ret = mtd_device_register(mtd, NULL, 0);
2814 if (ret) {
2815 dev_err(cdns_ctrl->dev,
2816 "failed to register mtd device: %d\n", ret);
2817 nand_cleanup(chip);
2818 return ret;
2819 }
2820
2821 list_add_tail(&cdns_chip->node, &cdns_ctrl->chips);
2822
2823 return 0;
2824}
2825
2826static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
2827{
2828 struct cdns_nand_chip *entry, *temp;
2829 struct nand_chip *chip;
2830 int ret;
2831
2832 list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
2833 chip = &entry->chip;
2834 ret = mtd_device_unregister(nand_to_mtd(chip));
2835 WARN_ON(ret);
2836 nand_cleanup(chip);
2837 list_del(&entry->node);
2838 }
2839}
2840
2841static int cadence_nand_chips_init(struct cdns_nand_ctrl *cdns_ctrl)
2842{
2843 struct device_node *np = cdns_ctrl->dev->of_node;
2844 struct device_node *nand_np;
2845 int max_cs = cdns_ctrl->caps2.max_banks;
2846 int nchips, ret;
2847
2848 nchips = of_get_child_count(np);
2849
2850 if (nchips > max_cs) {
2851 dev_err(cdns_ctrl->dev,
2852 "too many NAND chips: %d (max = %d CS)\n",
2853 nchips, max_cs);
2854 return -EINVAL;
2855 }
2856
2857 for_each_child_of_node(np, nand_np) {
2858 ret = cadence_nand_chip_init(cdns_ctrl, nand_np);
2859 if (ret) {
2860 of_node_put(nand_np);
2861 cadence_nand_chips_cleanup(cdns_ctrl);
2862 return ret;
2863 }
2864 }
2865
2866 return 0;
2867}
2868
2869static void
2870cadence_nand_irq_cleanup(int irqnum, struct cdns_nand_ctrl *cdns_ctrl)
2871{
2872 /* Disable interrupts. */
2873 writel_relaxed(INTR_ENABLE_INTR_EN, cdns_ctrl->reg + INTR_ENABLE);
2874}
2875
2876static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
2877{
2878 dma_cap_mask_t mask;
2879 int ret;
2880
2881 cdns_ctrl->cdma_desc = dma_alloc_coherent(cdns_ctrl->dev,
2882 sizeof(*cdns_ctrl->cdma_desc),
2883 &cdns_ctrl->dma_cdma_desc,
2884 GFP_KERNEL);
2885 if (!cdns_ctrl->dma_cdma_desc)
2886 return -ENOMEM;
2887
2888 cdns_ctrl->buf_size = SZ_16K;
2889 cdns_ctrl->buf = kmalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2890 if (!cdns_ctrl->buf) {
2891 ret = -ENOMEM;
2892 goto free_buf_desc;
2893 }
2894
2895 if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
2896 IRQF_SHARED, "cadence-nand-controller",
2897 cdns_ctrl)) {
2898 dev_err(cdns_ctrl->dev, "Unable to allocate IRQ\n");
2899 ret = -ENODEV;
2900 goto free_buf;
2901 }
2902
2903 spin_lock_init(&cdns_ctrl->irq_lock);
2904 init_completion(&cdns_ctrl->complete);
2905
2906 ret = cadence_nand_hw_init(cdns_ctrl);
2907 if (ret)
2908 goto disable_irq;
2909
2910 dma_cap_zero(mask);
2911 dma_cap_set(DMA_MEMCPY, mask);
2912
2913 if (cdns_ctrl->caps1->has_dma) {
2914 cdns_ctrl->dmac = dma_request_channel(mask, NULL, NULL);
2915 if (!cdns_ctrl->dmac) {
2916 dev_err(cdns_ctrl->dev,
2917 "Unable to get a DMA channel\n");
2918 ret = -EBUSY;
2919 goto disable_irq;
2920 }
2921 }
2922
2923 nand_controller_init(&cdns_ctrl->controller);
2924 INIT_LIST_HEAD(&cdns_ctrl->chips);
2925
2926 cdns_ctrl->controller.ops = &cadence_nand_controller_ops;
2927 cdns_ctrl->curr_corr_str_idx = 0xFF;
2928
2929 ret = cadence_nand_chips_init(cdns_ctrl);
2930 if (ret) {
2931 dev_err(cdns_ctrl->dev, "Failed to register MTD: %d\n",
2932 ret);
2933 goto dma_release_chnl;
2934 }
2935
2936 kfree(cdns_ctrl->buf);
2937 cdns_ctrl->buf = kzalloc(cdns_ctrl->buf_size, GFP_KERNEL);
2938 if (!cdns_ctrl->buf) {
2939 ret = -ENOMEM;
2940 goto dma_release_chnl;
2941 }
2942
2943 return 0;
2944
2945dma_release_chnl:
2946 if (cdns_ctrl->dmac)
2947 dma_release_channel(cdns_ctrl->dmac);
2948
2949disable_irq:
2950 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2951
2952free_buf:
2953 kfree(cdns_ctrl->buf);
2954
2955free_buf_desc:
2956 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2957 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2958
2959 return ret;
2960}
2961
2962/* Driver exit point. */
2963static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
2964{
2965 cadence_nand_chips_cleanup(cdns_ctrl);
2966 cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
2967 kfree(cdns_ctrl->buf);
2968 dma_free_coherent(cdns_ctrl->dev, sizeof(struct cadence_nand_cdma_desc),
2969 cdns_ctrl->cdma_desc, cdns_ctrl->dma_cdma_desc);
2970
2971 if (cdns_ctrl->dmac)
2972 dma_release_channel(cdns_ctrl->dmac);
2973}
2974
2975struct cadence_nand_dt {
2976 struct cdns_nand_ctrl cdns_ctrl;
2977 struct clk *clk;
2978};
2979
2980static const struct cadence_nand_dt_devdata cadence_nand_default = {
2981 .if_skew = 0,
2982 .has_dma = 1,
2983};
2984
2985static const struct of_device_id cadence_nand_dt_ids[] = {
2986 {
2987 .compatible = "cdns,hp-nfc",
2988 .data = &cadence_nand_default
2989 }, {}
2990};
2991
2992MODULE_DEVICE_TABLE(of, cadence_nand_dt_ids);
2993
2994static int cadence_nand_dt_probe(struct platform_device *ofdev)
2995{
2996 struct resource *res;
2997 struct cadence_nand_dt *dt;
2998 struct cdns_nand_ctrl *cdns_ctrl;
2999 int ret;
3000 const struct cadence_nand_dt_devdata *devdata;
3001 u32 val;
3002
3003 devdata = device_get_match_data(&ofdev->dev);
3004 if (!devdata) {
3005 pr_err("Failed to find the right device id.\n");
3006 return -ENOMEM;
3007 }
3008
3009 dt = devm_kzalloc(&ofdev->dev, sizeof(*dt), GFP_KERNEL);
3010 if (!dt)
3011 return -ENOMEM;
3012
3013 cdns_ctrl = &dt->cdns_ctrl;
3014 cdns_ctrl->caps1 = devdata;
3015
3016 cdns_ctrl->dev = &ofdev->dev;
3017 cdns_ctrl->irq = platform_get_irq(ofdev, 0);
3018 if (cdns_ctrl->irq < 0)
3019 return cdns_ctrl->irq;
3020
3021 dev_info(cdns_ctrl->dev, "IRQ: nr %d\n", cdns_ctrl->irq);
3022
3023 cdns_ctrl->reg = devm_platform_ioremap_resource(ofdev, 0);
3024 if (IS_ERR(cdns_ctrl->reg))
3025 return PTR_ERR(cdns_ctrl->reg);
3026
3027 cdns_ctrl->io.virt = devm_platform_get_and_ioremap_resource(ofdev, 1, &res);
3028 if (IS_ERR(cdns_ctrl->io.virt))
3029 return PTR_ERR(cdns_ctrl->io.virt);
3030 cdns_ctrl->io.dma = res->start;
3031
3032 dt->clk = devm_clk_get(cdns_ctrl->dev, "nf_clk");
3033 if (IS_ERR(dt->clk))
3034 return PTR_ERR(dt->clk);
3035
3036 cdns_ctrl->nf_clk_rate = clk_get_rate(dt->clk);
3037
3038 ret = of_property_read_u32(ofdev->dev.of_node,
3039 "cdns,board-delay-ps", &val);
3040 if (ret) {
3041 val = 4830;
3042 dev_info(cdns_ctrl->dev,
3043 "missing cdns,board-delay-ps property, %d was set\n",
3044 val);
3045 }
3046 cdns_ctrl->board_delay = val;
3047
3048 ret = cadence_nand_init(cdns_ctrl);
3049 if (ret)
3050 return ret;
3051
3052 platform_set_drvdata(ofdev, dt);
3053 return 0;
3054}
3055
3056static void cadence_nand_dt_remove(struct platform_device *ofdev)
3057{
3058 struct cadence_nand_dt *dt = platform_get_drvdata(ofdev);
3059
3060 cadence_nand_remove(&dt->cdns_ctrl);
3061}
3062
3063static struct platform_driver cadence_nand_dt_driver = {
3064 .probe = cadence_nand_dt_probe,
3065 .remove_new = cadence_nand_dt_remove,
3066 .driver = {
3067 .name = "cadence-nand-controller",
3068 .of_match_table = cadence_nand_dt_ids,
3069 },
3070};
3071
3072module_platform_driver(cadence_nand_dt_driver);
3073
3074MODULE_AUTHOR("Piotr Sroka <piotrs@cadence.com>");
3075MODULE_LICENSE("GPL v2");
3076MODULE_DESCRIPTION("Driver for Cadence NAND flash controller");
3077