Loading...
1// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2/*
3 * Amlogic Meson Nand Flash Controller Driver
4 *
5 * Copyright (c) 2018 Amlogic, inc.
6 * Author: Liang Yang <liang.yang@amlogic.com>
7 */
8
9#include <linux/platform_device.h>
10#include <linux/dma-mapping.h>
11#include <linux/interrupt.h>
12#include <linux/clk.h>
13#include <linux/clk-provider.h>
14#include <linux/mtd/rawnand.h>
15#include <linux/mtd/mtd.h>
16#include <linux/mfd/syscon.h>
17#include <linux/regmap.h>
18#include <linux/slab.h>
19#include <linux/module.h>
20#include <linux/iopoll.h>
21#include <linux/of.h>
22#include <linux/sched/task_stack.h>
23
24#define NFC_REG_CMD 0x00
25#define NFC_CMD_IDLE (0xc << 14)
26#define NFC_CMD_CLE (0x5 << 14)
27#define NFC_CMD_ALE (0x6 << 14)
28#define NFC_CMD_ADL ((0 << 16) | (3 << 20))
29#define NFC_CMD_ADH ((1 << 16) | (3 << 20))
30#define NFC_CMD_AIL ((2 << 16) | (3 << 20))
31#define NFC_CMD_AIH ((3 << 16) | (3 << 20))
32#define NFC_CMD_SEED ((8 << 16) | (3 << 20))
33#define NFC_CMD_M2N ((0 << 17) | (2 << 20))
34#define NFC_CMD_N2M ((1 << 17) | (2 << 20))
35#define NFC_CMD_RB BIT(20)
36#define NFC_CMD_SCRAMBLER_ENABLE BIT(19)
37#define NFC_CMD_SCRAMBLER_DISABLE 0
38#define NFC_CMD_SHORTMODE_ENABLE 1
39#define NFC_CMD_SHORTMODE_DISABLE 0
40#define NFC_CMD_RB_INT BIT(14)
41#define NFC_CMD_RB_INT_NO_PIN ((0xb << 10) | BIT(18) | BIT(16))
42
43#define NFC_CMD_GET_SIZE(x) (((x) >> 22) & GENMASK(4, 0))
44
45#define NFC_REG_CFG 0x04
46#define NFC_REG_DADR 0x08
47#define NFC_REG_IADR 0x0c
48#define NFC_REG_BUF 0x10
49#define NFC_REG_INFO 0x14
50#define NFC_REG_DC 0x18
51#define NFC_REG_ADR 0x1c
52#define NFC_REG_DL 0x20
53#define NFC_REG_DH 0x24
54#define NFC_REG_CADR 0x28
55#define NFC_REG_SADR 0x2c
56#define NFC_REG_PINS 0x30
57#define NFC_REG_VER 0x38
58
59#define NFC_RB_IRQ_EN BIT(21)
60
61#define CLK_DIV_SHIFT 0
62#define CLK_DIV_WIDTH 6
63
64#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \
65 ( \
66 (cmd_dir) | \
67 (ran) | \
68 ((bch) << 14) | \
69 ((short_mode) << 13) | \
70 (((page_size) & 0x7f) << 6) | \
71 ((pages) & 0x3f) \
72 )
73
74#define GENCMDDADDRL(adl, addr) ((adl) | ((addr) & 0xffff))
75#define GENCMDDADDRH(adh, addr) ((adh) | (((addr) >> 16) & 0xffff))
76#define GENCMDIADDRL(ail, addr) ((ail) | ((addr) & 0xffff))
77#define GENCMDIADDRH(aih, addr) ((aih) | (((addr) >> 16) & 0xffff))
78
79#define DMA_DIR(dir) ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
80#define DMA_ADDR_ALIGN 8
81
82#define NFC_SHORT_MODE_ECC_SZ 384
83
84#define ECC_CHECK_RETURN_FF (-1)
85
86#define NAND_CE0 (0xe << 10)
87#define NAND_CE1 (0xd << 10)
88
89#define DMA_BUSY_TIMEOUT 0x100000
90#define CMD_FIFO_EMPTY_TIMEOUT 1000
91
92#define MAX_CE_NUM 2
93
94/* eMMC clock register, misc control */
95#define CLK_SELECT_NAND BIT(31)
96#define CLK_ALWAYS_ON_NAND BIT(24)
97#define CLK_SELECT_FIX_PLL2 BIT(6)
98
99#define NFC_CLK_CYCLE 6
100
101/* nand flash controller delay 3 ns */
102#define NFC_DEFAULT_DELAY 3000
103
104#define ROW_ADDER(page, index) (((page) >> (8 * (index))) & 0xff)
105#define MAX_CYCLE_ADDRS 5
106#define DIRREAD 1
107#define DIRWRITE 0
108
109#define ECC_PARITY_BCH8_512B 14
110#define ECC_COMPLETE BIT(31)
111#define ECC_ERR_CNT(x) (((x) >> 24) & GENMASK(5, 0))
112#define ECC_ZERO_CNT(x) (((x) >> 16) & GENMASK(5, 0))
113#define ECC_UNCORRECTABLE 0x3f
114
115#define PER_INFO_BYTE 8
116
117#define NFC_CMD_RAW_LEN GENMASK(13, 0)
118
119#define NFC_COLUMN_ADDR_0 0
120#define NFC_COLUMN_ADDR_1 0
121
122struct meson_nfc_nand_chip {
123 struct list_head node;
124 struct nand_chip nand;
125 unsigned long clk_rate;
126 unsigned long level1_divider;
127 u32 bus_timing;
128 u32 twb;
129 u32 tadl;
130 u32 tbers_max;
131 u32 boot_pages;
132 u32 boot_page_step;
133
134 u32 bch_mode;
135 u8 *data_buf;
136 __le64 *info_buf;
137 u32 nsels;
138 u8 sels[] __counted_by(nsels);
139};
140
141struct meson_nand_ecc {
142 u32 bch;
143 u32 strength;
144 u32 size;
145};
146
147struct meson_nfc_data {
148 const struct nand_ecc_caps *ecc_caps;
149};
150
151struct meson_nfc_param {
152 u32 chip_select;
153 u32 rb_select;
154};
155
156struct nand_rw_cmd {
157 u32 cmd0;
158 u32 addrs[MAX_CYCLE_ADDRS];
159 u32 cmd1;
160};
161
162struct nand_timing {
163 u32 twb;
164 u32 tadl;
165 u32 tbers_max;
166};
167
168struct meson_nfc {
169 struct nand_controller controller;
170 struct clk *core_clk;
171 struct clk *device_clk;
172 struct clk *nand_clk;
173 struct clk_divider nand_divider;
174
175 unsigned long clk_rate;
176 u32 bus_timing;
177
178 struct device *dev;
179 void __iomem *reg_base;
180 void __iomem *reg_clk;
181 struct completion completion;
182 struct list_head chips;
183 const struct meson_nfc_data *data;
184 struct meson_nfc_param param;
185 struct nand_timing timing;
186 union {
187 int cmd[32];
188 struct nand_rw_cmd rw;
189 } cmdfifo;
190
191 dma_addr_t daddr;
192 dma_addr_t iaddr;
193 u32 info_bytes;
194
195 unsigned long assigned_cs;
196 bool no_rb_pin;
197};
198
199enum {
200 NFC_ECC_BCH8_512 = 1,
201 NFC_ECC_BCH8_1K,
202 NFC_ECC_BCH24_1K,
203 NFC_ECC_BCH30_1K,
204 NFC_ECC_BCH40_1K,
205 NFC_ECC_BCH50_1K,
206 NFC_ECC_BCH60_1K,
207};
208
209#define MESON_ECC_DATA(b, s, sz) { .bch = (b), .strength = (s), .size = (sz) }
210
211static struct meson_nand_ecc meson_ecc[] = {
212 MESON_ECC_DATA(NFC_ECC_BCH8_512, 8, 512),
213 MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8, 1024),
214 MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24, 1024),
215 MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30, 1024),
216 MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40, 1024),
217 MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50, 1024),
218 MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60, 1024),
219};
220
221static int meson_nand_calc_ecc_bytes(int step_size, int strength)
222{
223 int ecc_bytes;
224
225 if (step_size == 512 && strength == 8)
226 return ECC_PARITY_BCH8_512B;
227
228 ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
229 ecc_bytes = ALIGN(ecc_bytes, 2);
230
231 return ecc_bytes;
232}
233
234NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
235 meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
236
237static const int axg_stepinfo_strengths[] = { 8 };
238
239static const struct nand_ecc_step_info axg_stepinfo[] = {
240 {
241 .stepsize = 1024,
242 .strengths = axg_stepinfo_strengths,
243 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
244 },
245 {
246 .stepsize = 512,
247 .strengths = axg_stepinfo_strengths,
248 .nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
249 },
250};
251
252static const struct nand_ecc_caps meson_axg_ecc_caps = {
253 .stepinfos = axg_stepinfo,
254 .nstepinfos = ARRAY_SIZE(axg_stepinfo),
255 .calc_ecc_bytes = meson_nand_calc_ecc_bytes,
256};
257
258static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
259{
260 return container_of(nand, struct meson_nfc_nand_chip, nand);
261}
262
263static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
264{
265 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
266 struct meson_nfc *nfc = nand_get_controller_data(nand);
267 int ret, value;
268
269 if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
270 return;
271
272 nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
273 nfc->param.rb_select = nfc->param.chip_select;
274 nfc->timing.twb = meson_chip->twb;
275 nfc->timing.tadl = meson_chip->tadl;
276 nfc->timing.tbers_max = meson_chip->tbers_max;
277
278 if (nfc->clk_rate != meson_chip->clk_rate) {
279 ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
280 if (ret) {
281 dev_err(nfc->dev, "failed to set clock rate\n");
282 return;
283 }
284 nfc->clk_rate = meson_chip->clk_rate;
285 }
286 if (nfc->bus_timing != meson_chip->bus_timing) {
287 value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
288 writel(value, nfc->reg_base + NFC_REG_CFG);
289 writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
290 nfc->bus_timing = meson_chip->bus_timing;
291 }
292}
293
294static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
295{
296 writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
297 nfc->reg_base + NFC_REG_CMD);
298}
299
300static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
301{
302 writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
303 nfc->reg_base + NFC_REG_CMD);
304}
305
306static int meson_nfc_is_boot_page(struct nand_chip *nand, int page)
307{
308 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
309
310 return (nand->options & NAND_IS_BOOT_MEDIUM) &&
311 !(page % meson_chip->boot_page_step) &&
312 (page < meson_chip->boot_pages);
313}
314
315static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir, int page)
316{
317 const struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
318 struct mtd_info *mtd = nand_to_mtd(nand);
319 struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
320 int len = mtd->writesize, pagesize, pages;
321 int scrambler;
322 u32 cmd;
323
324 if (nand->options & NAND_NEED_SCRAMBLING)
325 scrambler = NFC_CMD_SCRAMBLER_ENABLE;
326 else
327 scrambler = NFC_CMD_SCRAMBLER_DISABLE;
328
329 if (raw) {
330 len = mtd->writesize + mtd->oobsize;
331 cmd = len | scrambler | DMA_DIR(dir);
332 } else if (meson_nfc_is_boot_page(nand, page)) {
333 pagesize = NFC_SHORT_MODE_ECC_SZ >> 3;
334 pages = mtd->writesize / 512;
335
336 scrambler = NFC_CMD_SCRAMBLER_ENABLE;
337 cmd = CMDRWGEN(DMA_DIR(dir), scrambler, NFC_ECC_BCH8_1K,
338 NFC_CMD_SHORTMODE_ENABLE, pagesize, pages);
339 } else {
340 pagesize = nand->ecc.size >> 3;
341 pages = len / nand->ecc.size;
342
343 cmd = CMDRWGEN(DMA_DIR(dir), scrambler, meson_chip->bch_mode,
344 NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
345 }
346
347 if (scrambler == NFC_CMD_SCRAMBLER_ENABLE)
348 meson_nfc_cmd_seed(nfc, page);
349
350 writel(cmd, nfc->reg_base + NFC_REG_CMD);
351}
352
353static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
354{
355 /*
356 * Insert two commands to make sure all valid commands are finished.
357 *
358 * The Nand flash controller is designed as two stages pipleline -
359 * a) fetch and b) excute.
360 * There might be cases when the driver see command queue is empty,
361 * but the Nand flash controller still has two commands buffered,
362 * one is fetched into NFC request queue (ready to run), and another
363 * is actively executing. So pushing 2 "IDLE" commands guarantees that
364 * the pipeline is emptied.
365 */
366 meson_nfc_cmd_idle(nfc, 0);
367 meson_nfc_cmd_idle(nfc, 0);
368}
369
370static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
371 unsigned int timeout_ms)
372{
373 u32 cmd_size = 0;
374 int ret;
375
376 /* wait cmd fifo is empty */
377 ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
378 !NFC_CMD_GET_SIZE(cmd_size),
379 10, timeout_ms * 1000);
380 if (ret)
381 dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
382
383 return ret;
384}
385
386static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
387{
388 meson_nfc_drain_cmd(nfc);
389
390 return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
391}
392
393static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
394{
395 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
396 int len;
397
398 len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
399
400 return meson_chip->data_buf + len;
401}
402
403static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
404{
405 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
406 int len, temp;
407
408 temp = nand->ecc.size + nand->ecc.bytes;
409 len = (temp + 2) * i;
410
411 return meson_chip->data_buf + len;
412}
413
414static void meson_nfc_get_data_oob(struct nand_chip *nand,
415 u8 *buf, u8 *oobbuf)
416{
417 int i, oob_len = 0;
418 u8 *dsrc, *osrc;
419
420 oob_len = nand->ecc.bytes + 2;
421 for (i = 0; i < nand->ecc.steps; i++) {
422 if (buf) {
423 dsrc = meson_nfc_data_ptr(nand, i);
424 memcpy(buf, dsrc, nand->ecc.size);
425 buf += nand->ecc.size;
426 }
427 osrc = meson_nfc_oob_ptr(nand, i);
428 memcpy(oobbuf, osrc, oob_len);
429 oobbuf += oob_len;
430 }
431}
432
433static void meson_nfc_set_data_oob(struct nand_chip *nand,
434 const u8 *buf, u8 *oobbuf)
435{
436 int i, oob_len = 0;
437 u8 *dsrc, *osrc;
438
439 oob_len = nand->ecc.bytes + 2;
440 for (i = 0; i < nand->ecc.steps; i++) {
441 if (buf) {
442 dsrc = meson_nfc_data_ptr(nand, i);
443 memcpy(dsrc, buf, nand->ecc.size);
444 buf += nand->ecc.size;
445 }
446 osrc = meson_nfc_oob_ptr(nand, i);
447 memcpy(osrc, oobbuf, oob_len);
448 oobbuf += oob_len;
449 }
450}
451
452static int meson_nfc_wait_no_rb_pin(struct nand_chip *nand, int timeout_ms,
453 bool need_cmd_read0)
454{
455 struct meson_nfc *nfc = nand_get_controller_data(nand);
456 u32 cmd, cfg;
457
458 meson_nfc_cmd_idle(nfc, nfc->timing.twb);
459 meson_nfc_drain_cmd(nfc);
460 meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
461
462 cfg = readl(nfc->reg_base + NFC_REG_CFG);
463 cfg |= NFC_RB_IRQ_EN;
464 writel(cfg, nfc->reg_base + NFC_REG_CFG);
465
466 reinit_completion(&nfc->completion);
467 nand_status_op(nand, NULL);
468
469 /* use the max erase time as the maximum clock for waiting R/B */
470 cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
471 writel(cmd, nfc->reg_base + NFC_REG_CMD);
472
473 if (!wait_for_completion_timeout(&nfc->completion,
474 msecs_to_jiffies(timeout_ms)))
475 return -ETIMEDOUT;
476
477 if (need_cmd_read0)
478 nand_exit_status_op(nand);
479
480 return 0;
481}
482
483static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
484{
485 u32 cmd, cfg;
486 int ret = 0;
487
488 meson_nfc_cmd_idle(nfc, nfc->timing.twb);
489 meson_nfc_drain_cmd(nfc);
490 meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
491
492 cfg = readl(nfc->reg_base + NFC_REG_CFG);
493 cfg |= NFC_RB_IRQ_EN;
494 writel(cfg, nfc->reg_base + NFC_REG_CFG);
495
496 reinit_completion(&nfc->completion);
497
498 /* use the max erase time as the maximum clock for waiting R/B */
499 cmd = NFC_CMD_RB | NFC_CMD_RB_INT
500 | nfc->param.chip_select | nfc->timing.tbers_max;
501 writel(cmd, nfc->reg_base + NFC_REG_CMD);
502
503 ret = wait_for_completion_timeout(&nfc->completion,
504 msecs_to_jiffies(timeout_ms));
505 if (ret == 0)
506 ret = -1;
507
508 return ret;
509}
510
511static int meson_nfc_queue_rb(struct nand_chip *nand, int timeout_ms,
512 bool need_cmd_read0)
513{
514 struct meson_nfc *nfc = nand_get_controller_data(nand);
515
516 if (nfc->no_rb_pin) {
517 /* This mode is used when there is no wired R/B pin.
518 * It works like 'nand_soft_waitrdy()', but instead of
519 * polling NAND_CMD_STATUS bit in the software loop,
520 * it will wait for interrupt - controllers checks IO
521 * bus and when it detects NAND_CMD_STATUS on it, it
522 * raises interrupt. After interrupt, NAND_CMD_READ0 is
523 * sent as terminator of the ready waiting procedure if
524 * needed (for all cases except page programming - this
525 * is reason of 'need_cmd_read0' flag).
526 */
527 return meson_nfc_wait_no_rb_pin(nand, timeout_ms,
528 need_cmd_read0);
529 } else {
530 return meson_nfc_wait_rb_pin(nfc, timeout_ms);
531 }
532}
533
534static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
535{
536 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
537 __le64 *info;
538 int i, count;
539
540 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
541 info = &meson_chip->info_buf[i];
542 *info |= oob_buf[count];
543 *info |= oob_buf[count + 1] << 8;
544 }
545}
546
547static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
548{
549 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
550 __le64 *info;
551 int i, count;
552
553 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
554 info = &meson_chip->info_buf[i];
555 oob_buf[count] = *info;
556 oob_buf[count + 1] = *info >> 8;
557 }
558}
559
560static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
561 u64 *correct_bitmap)
562{
563 struct mtd_info *mtd = nand_to_mtd(nand);
564 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
565 __le64 *info;
566 int ret = 0, i;
567
568 for (i = 0; i < nand->ecc.steps; i++) {
569 info = &meson_chip->info_buf[i];
570 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
571 mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
572 *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
573 *correct_bitmap |= BIT_ULL(i);
574 continue;
575 }
576 if ((nand->options & NAND_NEED_SCRAMBLING) &&
577 ECC_ZERO_CNT(*info) < nand->ecc.strength) {
578 mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
579 *bitflips = max_t(u32, *bitflips,
580 ECC_ZERO_CNT(*info));
581 ret = ECC_CHECK_RETURN_FF;
582 } else {
583 ret = -EBADMSG;
584 }
585 }
586 return ret;
587}
588
589static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
590 int datalen, void *infobuf, int infolen,
591 enum dma_data_direction dir)
592{
593 struct meson_nfc *nfc = nand_get_controller_data(nand);
594 u32 cmd;
595 int ret = 0;
596
597 nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
598 ret = dma_mapping_error(nfc->dev, nfc->daddr);
599 if (ret) {
600 dev_err(nfc->dev, "DMA mapping error\n");
601 return ret;
602 }
603 cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
604 writel(cmd, nfc->reg_base + NFC_REG_CMD);
605
606 cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
607 writel(cmd, nfc->reg_base + NFC_REG_CMD);
608
609 if (infobuf) {
610 nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
611 ret = dma_mapping_error(nfc->dev, nfc->iaddr);
612 if (ret) {
613 dev_err(nfc->dev, "DMA mapping error\n");
614 dma_unmap_single(nfc->dev,
615 nfc->daddr, datalen, dir);
616 return ret;
617 }
618 nfc->info_bytes = infolen;
619 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
620 writel(cmd, nfc->reg_base + NFC_REG_CMD);
621
622 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
623 writel(cmd, nfc->reg_base + NFC_REG_CMD);
624 }
625
626 return ret;
627}
628
629static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
630 int datalen, int infolen,
631 enum dma_data_direction dir)
632{
633 struct meson_nfc *nfc = nand_get_controller_data(nand);
634
635 dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
636 if (infolen) {
637 dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
638 nfc->info_bytes = 0;
639 }
640}
641
642static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
643{
644 struct meson_nfc *nfc = nand_get_controller_data(nand);
645 int ret = 0;
646 u32 cmd;
647 u8 *info;
648
649 info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
650 if (!info)
651 return -ENOMEM;
652
653 ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
654 PER_INFO_BYTE, DMA_FROM_DEVICE);
655 if (ret)
656 goto out;
657
658 cmd = NFC_CMD_N2M | len;
659 writel(cmd, nfc->reg_base + NFC_REG_CMD);
660
661 meson_nfc_drain_cmd(nfc);
662 meson_nfc_wait_cmd_finish(nfc, 1000);
663 meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
664
665out:
666 kfree(info);
667
668 return ret;
669}
670
671static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
672{
673 struct meson_nfc *nfc = nand_get_controller_data(nand);
674 int ret = 0;
675 u32 cmd;
676
677 ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
678 0, DMA_TO_DEVICE);
679 if (ret)
680 return ret;
681
682 cmd = NFC_CMD_M2N | len;
683 writel(cmd, nfc->reg_base + NFC_REG_CMD);
684
685 meson_nfc_drain_cmd(nfc);
686 meson_nfc_wait_cmd_finish(nfc, 1000);
687 meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
688
689 return ret;
690}
691
692static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
693 int page, bool in)
694{
695 const struct nand_sdr_timings *sdr =
696 nand_get_sdr_timings(nand_get_interface_config(nand));
697 struct mtd_info *mtd = nand_to_mtd(nand);
698 struct meson_nfc *nfc = nand_get_controller_data(nand);
699 u32 *addrs = nfc->cmdfifo.rw.addrs;
700 u32 cs = nfc->param.chip_select;
701 u32 cmd0, cmd_num, row_start;
702 int i;
703
704 cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
705
706 cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
707 nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
708
709 addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
710 if (mtd->writesize <= 512) {
711 cmd_num--;
712 row_start = 1;
713 } else {
714 addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
715 row_start = 2;
716 }
717
718 addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
719 addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
720
721 if (nand->options & NAND_ROW_ADDR_3)
722 addrs[row_start + 2] =
723 cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
724 else
725 cmd_num--;
726
727 /* subtract cmd1 */
728 cmd_num--;
729
730 for (i = 0; i < cmd_num; i++)
731 writel_relaxed(nfc->cmdfifo.cmd[i],
732 nfc->reg_base + NFC_REG_CMD);
733
734 if (in) {
735 nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
736 writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
737 meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tR_max), true);
738 } else {
739 meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
740 }
741
742 return 0;
743}
744
745static int meson_nfc_write_page_sub(struct nand_chip *nand,
746 int page, int raw)
747{
748 const struct nand_sdr_timings *sdr =
749 nand_get_sdr_timings(nand_get_interface_config(nand));
750 struct mtd_info *mtd = nand_to_mtd(nand);
751 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
752 struct meson_nfc *nfc = nand_get_controller_data(nand);
753 int data_len, info_len;
754 u32 cmd;
755 int ret;
756
757 meson_nfc_select_chip(nand, nand->cur_cs);
758
759 data_len = mtd->writesize + mtd->oobsize;
760 info_len = nand->ecc.steps * PER_INFO_BYTE;
761
762 ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
763 if (ret)
764 return ret;
765
766 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
767 data_len, meson_chip->info_buf,
768 info_len, DMA_TO_DEVICE);
769 if (ret)
770 return ret;
771
772 meson_nfc_cmd_access(nand, raw, DIRWRITE, page);
773
774 cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
775 writel(cmd, nfc->reg_base + NFC_REG_CMD);
776 meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tPROG_max), false);
777
778 meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
779
780 return ret;
781}
782
783static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
784 int oob_required, int page)
785{
786 u8 *oob_buf = nand->oob_poi;
787
788 meson_nfc_set_data_oob(nand, buf, oob_buf);
789
790 return meson_nfc_write_page_sub(nand, page, 1);
791}
792
793static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
794 const u8 *buf, int oob_required, int page)
795{
796 struct mtd_info *mtd = nand_to_mtd(nand);
797 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
798 u8 *oob_buf = nand->oob_poi;
799
800 memcpy(meson_chip->data_buf, buf, mtd->writesize);
801 memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
802 meson_nfc_set_user_byte(nand, oob_buf);
803
804 return meson_nfc_write_page_sub(nand, page, 0);
805}
806
807static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
808 struct nand_chip *nand, int raw)
809{
810 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
811 __le64 *info;
812 u32 neccpages;
813 int ret;
814
815 neccpages = raw ? 1 : nand->ecc.steps;
816 info = &meson_chip->info_buf[neccpages - 1];
817 do {
818 usleep_range(10, 15);
819 /* info is updated by nfc dma engine*/
820 smp_rmb();
821 dma_sync_single_for_cpu(nfc->dev, nfc->iaddr, nfc->info_bytes,
822 DMA_FROM_DEVICE);
823 ret = *info & ECC_COMPLETE;
824 } while (!ret);
825}
826
827static int meson_nfc_read_page_sub(struct nand_chip *nand,
828 int page, int raw)
829{
830 struct mtd_info *mtd = nand_to_mtd(nand);
831 struct meson_nfc *nfc = nand_get_controller_data(nand);
832 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
833 int data_len, info_len;
834 int ret;
835
836 meson_nfc_select_chip(nand, nand->cur_cs);
837
838 data_len = mtd->writesize + mtd->oobsize;
839 info_len = nand->ecc.steps * PER_INFO_BYTE;
840
841 ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
842 if (ret)
843 return ret;
844
845 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
846 data_len, meson_chip->info_buf,
847 info_len, DMA_FROM_DEVICE);
848 if (ret)
849 return ret;
850
851 meson_nfc_cmd_access(nand, raw, DIRREAD, page);
852
853 ret = meson_nfc_wait_dma_finish(nfc);
854 meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
855
856 meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
857
858 return ret;
859}
860
861static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
862 int oob_required, int page)
863{
864 u8 *oob_buf = nand->oob_poi;
865 int ret;
866
867 ret = meson_nfc_read_page_sub(nand, page, 1);
868 if (ret)
869 return ret;
870
871 meson_nfc_get_data_oob(nand, buf, oob_buf);
872
873 return 0;
874}
875
876static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
877 int oob_required, int page)
878{
879 struct mtd_info *mtd = nand_to_mtd(nand);
880 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
881 struct nand_ecc_ctrl *ecc = &nand->ecc;
882 u64 correct_bitmap = 0;
883 u32 bitflips = 0;
884 u8 *oob_buf = nand->oob_poi;
885 int ret, i;
886
887 ret = meson_nfc_read_page_sub(nand, page, 0);
888 if (ret)
889 return ret;
890
891 meson_nfc_get_user_byte(nand, oob_buf);
892 ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
893 if (ret == ECC_CHECK_RETURN_FF) {
894 if (buf)
895 memset(buf, 0xff, mtd->writesize);
896 memset(oob_buf, 0xff, mtd->oobsize);
897 } else if (ret < 0) {
898 if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
899 mtd->ecc_stats.failed++;
900 return bitflips;
901 }
902 ret = meson_nfc_read_page_raw(nand, buf, 0, page);
903 if (ret)
904 return ret;
905
906 for (i = 0; i < nand->ecc.steps ; i++) {
907 u8 *data = buf + i * ecc->size;
908 u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
909
910 if (correct_bitmap & BIT_ULL(i))
911 continue;
912 ret = nand_check_erased_ecc_chunk(data, ecc->size,
913 oob, ecc->bytes + 2,
914 NULL, 0,
915 ecc->strength);
916 if (ret < 0) {
917 mtd->ecc_stats.failed++;
918 } else {
919 mtd->ecc_stats.corrected += ret;
920 bitflips = max_t(u32, bitflips, ret);
921 }
922 }
923 } else if (buf && buf != meson_chip->data_buf) {
924 memcpy(buf, meson_chip->data_buf, mtd->writesize);
925 }
926
927 return bitflips;
928}
929
930static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
931{
932 return meson_nfc_read_page_raw(nand, NULL, 1, page);
933}
934
935static int meson_nfc_read_oob(struct nand_chip *nand, int page)
936{
937 return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
938}
939
940static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
941{
942 if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
943 return false;
944
945 if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
946 return true;
947 return false;
948}
949
950static void *
951meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
952{
953 if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
954 return NULL;
955
956 if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
957 return instr->ctx.data.buf.in;
958
959 return kzalloc(instr->ctx.data.len, GFP_KERNEL);
960}
961
962static void
963meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
964 void *buf)
965{
966 if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
967 WARN_ON(!buf))
968 return;
969
970 if (buf == instr->ctx.data.buf.in)
971 return;
972
973 memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
974 kfree(buf);
975}
976
977static void *
978meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
979{
980 if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
981 return NULL;
982
983 if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
984 return (void *)instr->ctx.data.buf.out;
985
986 return kmemdup(instr->ctx.data.buf.out,
987 instr->ctx.data.len, GFP_KERNEL);
988}
989
990static void
991meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
992 const void *buf)
993{
994 if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
995 WARN_ON(!buf))
996 return;
997
998 if (buf != instr->ctx.data.buf.out)
999 kfree(buf);
1000}
1001
1002static int meson_nfc_check_op(struct nand_chip *chip,
1003 const struct nand_operation *op)
1004{
1005 int op_id;
1006
1007 for (op_id = 0; op_id < op->ninstrs; op_id++) {
1008 const struct nand_op_instr *instr;
1009
1010 instr = &op->instrs[op_id];
1011
1012 switch (instr->type) {
1013 case NAND_OP_DATA_IN_INSTR:
1014 case NAND_OP_DATA_OUT_INSTR:
1015 if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
1016 return -ENOTSUPP;
1017
1018 break;
1019 default:
1020 break;
1021 }
1022 }
1023
1024 return 0;
1025}
1026
1027static int meson_nfc_exec_op(struct nand_chip *nand,
1028 const struct nand_operation *op, bool check_only)
1029{
1030 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1031 struct meson_nfc *nfc = nand_get_controller_data(nand);
1032 const struct nand_op_instr *instr = NULL;
1033 void *buf;
1034 u32 op_id, delay_idle, cmd;
1035 int err;
1036 int i;
1037
1038 err = meson_nfc_check_op(nand, op);
1039 if (err)
1040 return err;
1041
1042 if (check_only)
1043 return 0;
1044
1045 meson_nfc_select_chip(nand, op->cs);
1046 for (op_id = 0; op_id < op->ninstrs; op_id++) {
1047 instr = &op->instrs[op_id];
1048 delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1049 meson_chip->level1_divider *
1050 NFC_CLK_CYCLE);
1051 switch (instr->type) {
1052 case NAND_OP_CMD_INSTR:
1053 cmd = nfc->param.chip_select | NFC_CMD_CLE;
1054 cmd |= instr->ctx.cmd.opcode & 0xff;
1055 writel(cmd, nfc->reg_base + NFC_REG_CMD);
1056 meson_nfc_cmd_idle(nfc, delay_idle);
1057 break;
1058
1059 case NAND_OP_ADDR_INSTR:
1060 for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1061 cmd = nfc->param.chip_select | NFC_CMD_ALE;
1062 cmd |= instr->ctx.addr.addrs[i] & 0xff;
1063 writel(cmd, nfc->reg_base + NFC_REG_CMD);
1064 }
1065 meson_nfc_cmd_idle(nfc, delay_idle);
1066 break;
1067
1068 case NAND_OP_DATA_IN_INSTR:
1069 buf = meson_nand_op_get_dma_safe_input_buf(instr);
1070 if (!buf)
1071 return -ENOMEM;
1072 meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
1073 meson_nand_op_put_dma_safe_input_buf(instr, buf);
1074 break;
1075
1076 case NAND_OP_DATA_OUT_INSTR:
1077 buf = meson_nand_op_get_dma_safe_output_buf(instr);
1078 if (!buf)
1079 return -ENOMEM;
1080 meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
1081 meson_nand_op_put_dma_safe_output_buf(instr, buf);
1082 break;
1083
1084 case NAND_OP_WAITRDY_INSTR:
1085 meson_nfc_queue_rb(nand, instr->ctx.waitrdy.timeout_ms,
1086 true);
1087 if (instr->delay_ns)
1088 meson_nfc_cmd_idle(nfc, delay_idle);
1089 break;
1090 }
1091 }
1092 meson_nfc_wait_cmd_finish(nfc, 1000);
1093 return 0;
1094}
1095
1096static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1097 struct mtd_oob_region *oobregion)
1098{
1099 struct nand_chip *nand = mtd_to_nand(mtd);
1100
1101 if (section >= nand->ecc.steps)
1102 return -ERANGE;
1103
1104 oobregion->offset = 2 + (section * (2 + nand->ecc.bytes));
1105 oobregion->length = nand->ecc.bytes;
1106
1107 return 0;
1108}
1109
1110static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1111 struct mtd_oob_region *oobregion)
1112{
1113 struct nand_chip *nand = mtd_to_nand(mtd);
1114
1115 if (section >= nand->ecc.steps)
1116 return -ERANGE;
1117
1118 oobregion->offset = section * (2 + nand->ecc.bytes);
1119 oobregion->length = 2;
1120
1121 return 0;
1122}
1123
1124static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1125 .ecc = meson_ooblayout_ecc,
1126 .free = meson_ooblayout_free,
1127};
1128
1129static int meson_nfc_clk_init(struct meson_nfc *nfc)
1130{
1131 struct clk_parent_data nfc_divider_parent_data[1] = {0};
1132 struct clk_init_data init = {0};
1133 int ret;
1134
1135 /* request core clock */
1136 nfc->core_clk = devm_clk_get(nfc->dev, "core");
1137 if (IS_ERR(nfc->core_clk)) {
1138 dev_err(nfc->dev, "failed to get core clock\n");
1139 return PTR_ERR(nfc->core_clk);
1140 }
1141
1142 nfc->device_clk = devm_clk_get(nfc->dev, "device");
1143 if (IS_ERR(nfc->device_clk)) {
1144 dev_err(nfc->dev, "failed to get device clock\n");
1145 return PTR_ERR(nfc->device_clk);
1146 }
1147
1148 init.name = devm_kasprintf(nfc->dev,
1149 GFP_KERNEL, "%s#div",
1150 dev_name(nfc->dev));
1151 if (!init.name)
1152 return -ENOMEM;
1153
1154 init.ops = &clk_divider_ops;
1155 nfc_divider_parent_data[0].fw_name = "device";
1156 init.parent_data = nfc_divider_parent_data;
1157 init.num_parents = 1;
1158 nfc->nand_divider.reg = nfc->reg_clk;
1159 nfc->nand_divider.shift = CLK_DIV_SHIFT;
1160 nfc->nand_divider.width = CLK_DIV_WIDTH;
1161 nfc->nand_divider.hw.init = &init;
1162 nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1163 CLK_DIVIDER_ROUND_CLOSEST |
1164 CLK_DIVIDER_ALLOW_ZERO;
1165
1166 nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1167 if (IS_ERR(nfc->nand_clk))
1168 return PTR_ERR(nfc->nand_clk);
1169
1170 /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1171 writel(CLK_ALWAYS_ON_NAND | CLK_SELECT_NAND | CLK_SELECT_FIX_PLL2,
1172 nfc->reg_clk);
1173
1174 ret = clk_prepare_enable(nfc->core_clk);
1175 if (ret) {
1176 dev_err(nfc->dev, "failed to enable core clock\n");
1177 return ret;
1178 }
1179
1180 ret = clk_prepare_enable(nfc->device_clk);
1181 if (ret) {
1182 dev_err(nfc->dev, "failed to enable device clock\n");
1183 goto err_device_clk;
1184 }
1185
1186 ret = clk_prepare_enable(nfc->nand_clk);
1187 if (ret) {
1188 dev_err(nfc->dev, "pre enable NFC divider fail\n");
1189 goto err_nand_clk;
1190 }
1191
1192 ret = clk_set_rate(nfc->nand_clk, 24000000);
1193 if (ret)
1194 goto err_disable_clk;
1195
1196 return 0;
1197
1198err_disable_clk:
1199 clk_disable_unprepare(nfc->nand_clk);
1200err_nand_clk:
1201 clk_disable_unprepare(nfc->device_clk);
1202err_device_clk:
1203 clk_disable_unprepare(nfc->core_clk);
1204 return ret;
1205}
1206
1207static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1208{
1209 clk_disable_unprepare(nfc->nand_clk);
1210 clk_disable_unprepare(nfc->device_clk);
1211 clk_disable_unprepare(nfc->core_clk);
1212}
1213
1214static void meson_nfc_free_buffer(struct nand_chip *nand)
1215{
1216 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1217
1218 kfree(meson_chip->info_buf);
1219 kfree(meson_chip->data_buf);
1220}
1221
1222static int meson_chip_buffer_init(struct nand_chip *nand)
1223{
1224 struct mtd_info *mtd = nand_to_mtd(nand);
1225 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1226 u32 page_bytes, info_bytes, nsectors;
1227
1228 nsectors = mtd->writesize / nand->ecc.size;
1229
1230 page_bytes = mtd->writesize + mtd->oobsize;
1231 info_bytes = nsectors * PER_INFO_BYTE;
1232
1233 meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1234 if (!meson_chip->data_buf)
1235 return -ENOMEM;
1236
1237 meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1238 if (!meson_chip->info_buf) {
1239 kfree(meson_chip->data_buf);
1240 return -ENOMEM;
1241 }
1242
1243 return 0;
1244}
1245
1246static
1247int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1248 const struct nand_interface_config *conf)
1249{
1250 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1251 const struct nand_sdr_timings *timings;
1252 u32 div, bt_min, bt_max, tbers_clocks;
1253
1254 timings = nand_get_sdr_timings(conf);
1255 if (IS_ERR(timings))
1256 return -ENOTSUPP;
1257
1258 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1259 return 0;
1260
1261 div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1262 bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1263 bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1264 timings->tRC_min / 2) / div;
1265
1266 meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1267 div * NFC_CLK_CYCLE);
1268 meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1269 div * NFC_CLK_CYCLE);
1270 tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1271 div * NFC_CLK_CYCLE);
1272 meson_chip->tbers_max = ilog2(tbers_clocks);
1273 if (!is_power_of_2(tbers_clocks))
1274 meson_chip->tbers_max++;
1275
1276 bt_min = DIV_ROUND_UP(bt_min, 1000);
1277 bt_max = DIV_ROUND_UP(bt_max, 1000);
1278
1279 if (bt_max < bt_min)
1280 return -EINVAL;
1281
1282 meson_chip->level1_divider = div;
1283 meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1284 meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1285
1286 return 0;
1287}
1288
1289static int meson_nand_bch_mode(struct nand_chip *nand)
1290{
1291 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1292 int i;
1293
1294 if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1295 return -EINVAL;
1296
1297 for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1298 if (meson_ecc[i].strength == nand->ecc.strength &&
1299 meson_ecc[i].size == nand->ecc.size) {
1300 meson_chip->bch_mode = meson_ecc[i].bch;
1301 return 0;
1302 }
1303 }
1304
1305 return -EINVAL;
1306}
1307
1308static void meson_nand_detach_chip(struct nand_chip *nand)
1309{
1310 meson_nfc_free_buffer(nand);
1311}
1312
1313static int meson_nand_attach_chip(struct nand_chip *nand)
1314{
1315 struct meson_nfc *nfc = nand_get_controller_data(nand);
1316 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1317 struct mtd_info *mtd = nand_to_mtd(nand);
1318 int raw_writesize;
1319 int ret;
1320
1321 if (!mtd->name) {
1322 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1323 "%s:nand%d",
1324 dev_name(nfc->dev),
1325 meson_chip->sels[0]);
1326 if (!mtd->name)
1327 return -ENOMEM;
1328 }
1329
1330 raw_writesize = mtd->writesize + mtd->oobsize;
1331 if (raw_writesize > NFC_CMD_RAW_LEN) {
1332 dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1333 raw_writesize, NFC_CMD_RAW_LEN);
1334 return -EINVAL;
1335 }
1336
1337 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1338 nand->bbt_options |= NAND_BBT_NO_OOB;
1339
1340 nand->options |= NAND_NO_SUBPAGE_WRITE;
1341
1342 ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1343 mtd->oobsize - 2);
1344 if (ret) {
1345 dev_err(nfc->dev, "failed to ECC init\n");
1346 return -EINVAL;
1347 }
1348
1349 mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1350
1351 ret = meson_nand_bch_mode(nand);
1352 if (ret)
1353 return -EINVAL;
1354
1355 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1356 nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1357 nand->ecc.write_page = meson_nfc_write_page_hwecc;
1358 nand->ecc.write_oob_raw = nand_write_oob_std;
1359 nand->ecc.write_oob = nand_write_oob_std;
1360
1361 nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1362 nand->ecc.read_page = meson_nfc_read_page_hwecc;
1363 nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1364 nand->ecc.read_oob = meson_nfc_read_oob;
1365
1366 if (nand->options & NAND_BUSWIDTH_16) {
1367 dev_err(nfc->dev, "16bits bus width not supported");
1368 return -EINVAL;
1369 }
1370 ret = meson_chip_buffer_init(nand);
1371 if (ret)
1372 return -ENOMEM;
1373
1374 return ret;
1375}
1376
1377static const struct nand_controller_ops meson_nand_controller_ops = {
1378 .attach_chip = meson_nand_attach_chip,
1379 .detach_chip = meson_nand_detach_chip,
1380 .setup_interface = meson_nfc_setup_interface,
1381 .exec_op = meson_nfc_exec_op,
1382};
1383
1384static int
1385meson_nfc_nand_chip_init(struct device *dev,
1386 struct meson_nfc *nfc, struct device_node *np)
1387{
1388 struct meson_nfc_nand_chip *meson_chip;
1389 struct nand_chip *nand;
1390 struct mtd_info *mtd;
1391 int ret, i;
1392 u32 tmp, nsels;
1393 u32 nand_rb_val = 0;
1394
1395 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1396 if (!nsels || nsels > MAX_CE_NUM) {
1397 dev_err(dev, "invalid register property size\n");
1398 return -EINVAL;
1399 }
1400
1401 meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1402 GFP_KERNEL);
1403 if (!meson_chip)
1404 return -ENOMEM;
1405
1406 meson_chip->nsels = nsels;
1407
1408 for (i = 0; i < nsels; i++) {
1409 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1410 if (ret) {
1411 dev_err(dev, "could not retrieve register property: %d\n",
1412 ret);
1413 return ret;
1414 }
1415
1416 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1417 dev_err(dev, "CS %d already assigned\n", tmp);
1418 return -EINVAL;
1419 }
1420 }
1421
1422 nand = &meson_chip->nand;
1423 nand->controller = &nfc->controller;
1424 nand->controller->ops = &meson_nand_controller_ops;
1425 nand_set_flash_node(nand, np);
1426 nand_set_controller_data(nand, nfc);
1427
1428 nand->options |= NAND_USES_DMA;
1429 mtd = nand_to_mtd(nand);
1430 mtd->owner = THIS_MODULE;
1431 mtd->dev.parent = dev;
1432
1433 ret = of_property_read_u32(np, "nand-rb", &nand_rb_val);
1434 if (ret == -EINVAL)
1435 nfc->no_rb_pin = true;
1436 else if (ret)
1437 return ret;
1438
1439 if (nand_rb_val)
1440 return -EINVAL;
1441
1442 ret = nand_scan(nand, nsels);
1443 if (ret)
1444 return ret;
1445
1446 if (nand->options & NAND_IS_BOOT_MEDIUM) {
1447 ret = of_property_read_u32(np, "amlogic,boot-pages",
1448 &meson_chip->boot_pages);
1449 if (ret) {
1450 dev_err(dev, "could not retrieve 'amlogic,boot-pages' property: %d",
1451 ret);
1452 nand_cleanup(nand);
1453 return ret;
1454 }
1455
1456 ret = of_property_read_u32(np, "amlogic,boot-page-step",
1457 &meson_chip->boot_page_step);
1458 if (ret) {
1459 dev_err(dev, "could not retrieve 'amlogic,boot-page-step' property: %d",
1460 ret);
1461 nand_cleanup(nand);
1462 return ret;
1463 }
1464 }
1465
1466 ret = mtd_device_register(mtd, NULL, 0);
1467 if (ret) {
1468 dev_err(dev, "failed to register MTD device: %d\n", ret);
1469 nand_cleanup(nand);
1470 return ret;
1471 }
1472
1473 list_add_tail(&meson_chip->node, &nfc->chips);
1474
1475 return 0;
1476}
1477
1478static void meson_nfc_nand_chips_cleanup(struct meson_nfc *nfc)
1479{
1480 struct meson_nfc_nand_chip *meson_chip;
1481 struct mtd_info *mtd;
1482
1483 while (!list_empty(&nfc->chips)) {
1484 meson_chip = list_first_entry(&nfc->chips,
1485 struct meson_nfc_nand_chip, node);
1486 mtd = nand_to_mtd(&meson_chip->nand);
1487 WARN_ON(mtd_device_unregister(mtd));
1488
1489 nand_cleanup(&meson_chip->nand);
1490 list_del(&meson_chip->node);
1491 }
1492}
1493
1494static int meson_nfc_nand_chips_init(struct device *dev,
1495 struct meson_nfc *nfc)
1496{
1497 struct device_node *np = dev->of_node;
1498 int ret;
1499
1500 for_each_child_of_node_scoped(np, nand_np) {
1501 ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1502 if (ret) {
1503 meson_nfc_nand_chips_cleanup(nfc);
1504 return ret;
1505 }
1506 }
1507
1508 return 0;
1509}
1510
1511static irqreturn_t meson_nfc_irq(int irq, void *id)
1512{
1513 struct meson_nfc *nfc = id;
1514 u32 cfg;
1515
1516 cfg = readl(nfc->reg_base + NFC_REG_CFG);
1517 if (!(cfg & NFC_RB_IRQ_EN))
1518 return IRQ_NONE;
1519
1520 cfg &= ~(NFC_RB_IRQ_EN);
1521 writel(cfg, nfc->reg_base + NFC_REG_CFG);
1522
1523 complete(&nfc->completion);
1524 return IRQ_HANDLED;
1525}
1526
1527static const struct meson_nfc_data meson_gxl_data = {
1528 .ecc_caps = &meson_gxl_ecc_caps,
1529};
1530
1531static const struct meson_nfc_data meson_axg_data = {
1532 .ecc_caps = &meson_axg_ecc_caps,
1533};
1534
1535static const struct of_device_id meson_nfc_id_table[] = {
1536 {
1537 .compatible = "amlogic,meson-gxl-nfc",
1538 .data = &meson_gxl_data,
1539 }, {
1540 .compatible = "amlogic,meson-axg-nfc",
1541 .data = &meson_axg_data,
1542 },
1543 {}
1544};
1545MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1546
1547static int meson_nfc_probe(struct platform_device *pdev)
1548{
1549 struct device *dev = &pdev->dev;
1550 struct meson_nfc *nfc;
1551 int ret, irq;
1552
1553 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1554 if (!nfc)
1555 return -ENOMEM;
1556
1557 nfc->data = of_device_get_match_data(&pdev->dev);
1558 if (!nfc->data)
1559 return -ENODEV;
1560
1561 nand_controller_init(&nfc->controller);
1562 INIT_LIST_HEAD(&nfc->chips);
1563 init_completion(&nfc->completion);
1564
1565 nfc->dev = dev;
1566
1567 nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1568 if (IS_ERR(nfc->reg_base))
1569 return PTR_ERR(nfc->reg_base);
1570
1571 nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1572 if (IS_ERR(nfc->reg_clk))
1573 return PTR_ERR(nfc->reg_clk);
1574
1575 irq = platform_get_irq(pdev, 0);
1576 if (irq < 0)
1577 return -EINVAL;
1578
1579 ret = meson_nfc_clk_init(nfc);
1580 if (ret) {
1581 dev_err(dev, "failed to initialize NAND clock\n");
1582 return ret;
1583 }
1584
1585 writel(0, nfc->reg_base + NFC_REG_CFG);
1586 ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1587 if (ret) {
1588 dev_err(dev, "failed to request NFC IRQ\n");
1589 ret = -EINVAL;
1590 goto err_clk;
1591 }
1592
1593 ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1594 if (ret) {
1595 dev_err(dev, "failed to set DMA mask\n");
1596 goto err_clk;
1597 }
1598
1599 platform_set_drvdata(pdev, nfc);
1600
1601 ret = meson_nfc_nand_chips_init(dev, nfc);
1602 if (ret) {
1603 dev_err(dev, "failed to init NAND chips\n");
1604 goto err_clk;
1605 }
1606
1607 return 0;
1608err_clk:
1609 meson_nfc_disable_clk(nfc);
1610 return ret;
1611}
1612
1613static void meson_nfc_remove(struct platform_device *pdev)
1614{
1615 struct meson_nfc *nfc = platform_get_drvdata(pdev);
1616
1617 meson_nfc_nand_chips_cleanup(nfc);
1618
1619 meson_nfc_disable_clk(nfc);
1620}
1621
1622static struct platform_driver meson_nfc_driver = {
1623 .probe = meson_nfc_probe,
1624 .remove = meson_nfc_remove,
1625 .driver = {
1626 .name = "meson-nand",
1627 .of_match_table = meson_nfc_id_table,
1628 },
1629};
1630module_platform_driver(meson_nfc_driver);
1631
1632MODULE_LICENSE("Dual MIT/GPL");
1633MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1634MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2/*
3 * Amlogic Meson Nand Flash Controller Driver
4 *
5 * Copyright (c) 2018 Amlogic, inc.
6 * Author: Liang Yang <liang.yang@amlogic.com>
7 */
8
9#include <linux/platform_device.h>
10#include <linux/dma-mapping.h>
11#include <linux/interrupt.h>
12#include <linux/clk.h>
13#include <linux/clk-provider.h>
14#include <linux/mtd/rawnand.h>
15#include <linux/mtd/mtd.h>
16#include <linux/mfd/syscon.h>
17#include <linux/regmap.h>
18#include <linux/slab.h>
19#include <linux/module.h>
20#include <linux/iopoll.h>
21#include <linux/of.h>
22#include <linux/of_device.h>
23#include <linux/sched/task_stack.h>
24
25#define NFC_REG_CMD 0x00
26#define NFC_CMD_IDLE (0xc << 14)
27#define NFC_CMD_CLE (0x5 << 14)
28#define NFC_CMD_ALE (0x6 << 14)
29#define NFC_CMD_ADL ((0 << 16) | (3 << 20))
30#define NFC_CMD_ADH ((1 << 16) | (3 << 20))
31#define NFC_CMD_AIL ((2 << 16) | (3 << 20))
32#define NFC_CMD_AIH ((3 << 16) | (3 << 20))
33#define NFC_CMD_SEED ((8 << 16) | (3 << 20))
34#define NFC_CMD_M2N ((0 << 17) | (2 << 20))
35#define NFC_CMD_N2M ((1 << 17) | (2 << 20))
36#define NFC_CMD_RB BIT(20)
37#define NFC_CMD_SCRAMBLER_ENABLE BIT(19)
38#define NFC_CMD_SCRAMBLER_DISABLE 0
39#define NFC_CMD_SHORTMODE_DISABLE 0
40#define NFC_CMD_RB_INT BIT(14)
41
42#define NFC_CMD_GET_SIZE(x) (((x) >> 22) & GENMASK(4, 0))
43
44#define NFC_REG_CFG 0x04
45#define NFC_REG_DADR 0x08
46#define NFC_REG_IADR 0x0c
47#define NFC_REG_BUF 0x10
48#define NFC_REG_INFO 0x14
49#define NFC_REG_DC 0x18
50#define NFC_REG_ADR 0x1c
51#define NFC_REG_DL 0x20
52#define NFC_REG_DH 0x24
53#define NFC_REG_CADR 0x28
54#define NFC_REG_SADR 0x2c
55#define NFC_REG_PINS 0x30
56#define NFC_REG_VER 0x38
57
58#define NFC_RB_IRQ_EN BIT(21)
59
60#define CLK_DIV_SHIFT 0
61#define CLK_DIV_WIDTH 6
62
63#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \
64 ( \
65 (cmd_dir) | \
66 ((ran) << 19) | \
67 ((bch) << 14) | \
68 ((short_mode) << 13) | \
69 (((page_size) & 0x7f) << 6) | \
70 ((pages) & 0x3f) \
71 )
72
73#define GENCMDDADDRL(adl, addr) ((adl) | ((addr) & 0xffff))
74#define GENCMDDADDRH(adh, addr) ((adh) | (((addr) >> 16) & 0xffff))
75#define GENCMDIADDRL(ail, addr) ((ail) | ((addr) & 0xffff))
76#define GENCMDIADDRH(aih, addr) ((aih) | (((addr) >> 16) & 0xffff))
77
78#define DMA_DIR(dir) ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
79
80#define ECC_CHECK_RETURN_FF (-1)
81
82#define NAND_CE0 (0xe << 10)
83#define NAND_CE1 (0xd << 10)
84
85#define DMA_BUSY_TIMEOUT 0x100000
86#define CMD_FIFO_EMPTY_TIMEOUT 1000
87
88#define MAX_CE_NUM 2
89
90/* eMMC clock register, misc control */
91#define CLK_SELECT_NAND BIT(31)
92
93#define NFC_CLK_CYCLE 6
94
95/* nand flash controller delay 3 ns */
96#define NFC_DEFAULT_DELAY 3000
97
98#define ROW_ADDER(page, index) (((page) >> (8 * (index))) & 0xff)
99#define MAX_CYCLE_ADDRS 5
100#define DIRREAD 1
101#define DIRWRITE 0
102
103#define ECC_PARITY_BCH8_512B 14
104#define ECC_COMPLETE BIT(31)
105#define ECC_ERR_CNT(x) (((x) >> 24) & GENMASK(5, 0))
106#define ECC_ZERO_CNT(x) (((x) >> 16) & GENMASK(5, 0))
107#define ECC_UNCORRECTABLE 0x3f
108
109#define PER_INFO_BYTE 8
110
111struct meson_nfc_nand_chip {
112 struct list_head node;
113 struct nand_chip nand;
114 unsigned long clk_rate;
115 unsigned long level1_divider;
116 u32 bus_timing;
117 u32 twb;
118 u32 tadl;
119 u32 tbers_max;
120
121 u32 bch_mode;
122 u8 *data_buf;
123 __le64 *info_buf;
124 u32 nsels;
125 u8 sels[];
126};
127
128struct meson_nand_ecc {
129 u32 bch;
130 u32 strength;
131};
132
133struct meson_nfc_data {
134 const struct nand_ecc_caps *ecc_caps;
135};
136
137struct meson_nfc_param {
138 u32 chip_select;
139 u32 rb_select;
140};
141
142struct nand_rw_cmd {
143 u32 cmd0;
144 u32 addrs[MAX_CYCLE_ADDRS];
145 u32 cmd1;
146};
147
148struct nand_timing {
149 u32 twb;
150 u32 tadl;
151 u32 tbers_max;
152};
153
154struct meson_nfc {
155 struct nand_controller controller;
156 struct clk *core_clk;
157 struct clk *device_clk;
158 struct clk *nand_clk;
159 struct clk_divider nand_divider;
160
161 unsigned long clk_rate;
162 u32 bus_timing;
163
164 struct device *dev;
165 void __iomem *reg_base;
166 void __iomem *reg_clk;
167 struct completion completion;
168 struct list_head chips;
169 const struct meson_nfc_data *data;
170 struct meson_nfc_param param;
171 struct nand_timing timing;
172 union {
173 int cmd[32];
174 struct nand_rw_cmd rw;
175 } cmdfifo;
176
177 dma_addr_t daddr;
178 dma_addr_t iaddr;
179
180 unsigned long assigned_cs;
181};
182
183enum {
184 NFC_ECC_BCH8_1K = 2,
185 NFC_ECC_BCH24_1K,
186 NFC_ECC_BCH30_1K,
187 NFC_ECC_BCH40_1K,
188 NFC_ECC_BCH50_1K,
189 NFC_ECC_BCH60_1K,
190};
191
192#define MESON_ECC_DATA(b, s) { .bch = (b), .strength = (s)}
193
194static struct meson_nand_ecc meson_ecc[] = {
195 MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8),
196 MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24),
197 MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30),
198 MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40),
199 MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50),
200 MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60),
201};
202
203static int meson_nand_calc_ecc_bytes(int step_size, int strength)
204{
205 int ecc_bytes;
206
207 if (step_size == 512 && strength == 8)
208 return ECC_PARITY_BCH8_512B;
209
210 ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
211 ecc_bytes = ALIGN(ecc_bytes, 2);
212
213 return ecc_bytes;
214}
215
216NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
217 meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
218NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps,
219 meson_nand_calc_ecc_bytes, 1024, 8);
220
221static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
222{
223 return container_of(nand, struct meson_nfc_nand_chip, nand);
224}
225
226static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
227{
228 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
229 struct meson_nfc *nfc = nand_get_controller_data(nand);
230 int ret, value;
231
232 if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
233 return;
234
235 nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
236 nfc->param.rb_select = nfc->param.chip_select;
237 nfc->timing.twb = meson_chip->twb;
238 nfc->timing.tadl = meson_chip->tadl;
239 nfc->timing.tbers_max = meson_chip->tbers_max;
240
241 if (nfc->clk_rate != meson_chip->clk_rate) {
242 ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate);
243 if (ret) {
244 dev_err(nfc->dev, "failed to set clock rate\n");
245 return;
246 }
247 nfc->clk_rate = meson_chip->clk_rate;
248 }
249 if (nfc->bus_timing != meson_chip->bus_timing) {
250 value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
251 writel(value, nfc->reg_base + NFC_REG_CFG);
252 writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
253 nfc->bus_timing = meson_chip->bus_timing;
254 }
255}
256
257static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
258{
259 writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
260 nfc->reg_base + NFC_REG_CMD);
261}
262
263static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
264{
265 writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
266 nfc->reg_base + NFC_REG_CMD);
267}
268
269static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
270 int scrambler)
271{
272 struct mtd_info *mtd = nand_to_mtd(nand);
273 struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
274 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
275 u32 bch = meson_chip->bch_mode, cmd;
276 int len = mtd->writesize, pagesize, pages;
277
278 pagesize = nand->ecc.size;
279
280 if (raw) {
281 len = mtd->writesize + mtd->oobsize;
282 cmd = (len & GENMASK(5, 0)) | scrambler | DMA_DIR(dir);
283 writel(cmd, nfc->reg_base + NFC_REG_CMD);
284 return;
285 }
286
287 pages = len / nand->ecc.size;
288
289 cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
290 NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
291
292 writel(cmd, nfc->reg_base + NFC_REG_CMD);
293}
294
295static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
296{
297 /*
298 * Insert two commands to make sure all valid commands are finished.
299 *
300 * The Nand flash controller is designed as two stages pipleline -
301 * a) fetch and b) excute.
302 * There might be cases when the driver see command queue is empty,
303 * but the Nand flash controller still has two commands buffered,
304 * one is fetched into NFC request queue (ready to run), and another
305 * is actively executing. So pushing 2 "IDLE" commands guarantees that
306 * the pipeline is emptied.
307 */
308 meson_nfc_cmd_idle(nfc, 0);
309 meson_nfc_cmd_idle(nfc, 0);
310}
311
312static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
313 unsigned int timeout_ms)
314{
315 u32 cmd_size = 0;
316 int ret;
317
318 /* wait cmd fifo is empty */
319 ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
320 !NFC_CMD_GET_SIZE(cmd_size),
321 10, timeout_ms * 1000);
322 if (ret)
323 dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
324
325 return ret;
326}
327
328static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
329{
330 meson_nfc_drain_cmd(nfc);
331
332 return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
333}
334
335static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
336{
337 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
338 int len;
339
340 len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
341
342 return meson_chip->data_buf + len;
343}
344
345static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
346{
347 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
348 int len, temp;
349
350 temp = nand->ecc.size + nand->ecc.bytes;
351 len = (temp + 2) * i;
352
353 return meson_chip->data_buf + len;
354}
355
356static void meson_nfc_get_data_oob(struct nand_chip *nand,
357 u8 *buf, u8 *oobbuf)
358{
359 int i, oob_len = 0;
360 u8 *dsrc, *osrc;
361
362 oob_len = nand->ecc.bytes + 2;
363 for (i = 0; i < nand->ecc.steps; i++) {
364 if (buf) {
365 dsrc = meson_nfc_data_ptr(nand, i);
366 memcpy(buf, dsrc, nand->ecc.size);
367 buf += nand->ecc.size;
368 }
369 osrc = meson_nfc_oob_ptr(nand, i);
370 memcpy(oobbuf, osrc, oob_len);
371 oobbuf += oob_len;
372 }
373}
374
375static void meson_nfc_set_data_oob(struct nand_chip *nand,
376 const u8 *buf, u8 *oobbuf)
377{
378 int i, oob_len = 0;
379 u8 *dsrc, *osrc;
380
381 oob_len = nand->ecc.bytes + 2;
382 for (i = 0; i < nand->ecc.steps; i++) {
383 if (buf) {
384 dsrc = meson_nfc_data_ptr(nand, i);
385 memcpy(dsrc, buf, nand->ecc.size);
386 buf += nand->ecc.size;
387 }
388 osrc = meson_nfc_oob_ptr(nand, i);
389 memcpy(osrc, oobbuf, oob_len);
390 oobbuf += oob_len;
391 }
392}
393
394static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms)
395{
396 u32 cmd, cfg;
397 int ret = 0;
398
399 meson_nfc_cmd_idle(nfc, nfc->timing.twb);
400 meson_nfc_drain_cmd(nfc);
401 meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
402
403 cfg = readl(nfc->reg_base + NFC_REG_CFG);
404 cfg |= NFC_RB_IRQ_EN;
405 writel(cfg, nfc->reg_base + NFC_REG_CFG);
406
407 reinit_completion(&nfc->completion);
408
409 /* use the max erase time as the maximum clock for waiting R/B */
410 cmd = NFC_CMD_RB | NFC_CMD_RB_INT
411 | nfc->param.chip_select | nfc->timing.tbers_max;
412 writel(cmd, nfc->reg_base + NFC_REG_CMD);
413
414 ret = wait_for_completion_timeout(&nfc->completion,
415 msecs_to_jiffies(timeout_ms));
416 if (ret == 0)
417 ret = -1;
418
419 return ret;
420}
421
422static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
423{
424 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
425 __le64 *info;
426 int i, count;
427
428 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
429 info = &meson_chip->info_buf[i];
430 *info |= oob_buf[count];
431 *info |= oob_buf[count + 1] << 8;
432 }
433}
434
435static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
436{
437 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
438 __le64 *info;
439 int i, count;
440
441 for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
442 info = &meson_chip->info_buf[i];
443 oob_buf[count] = *info;
444 oob_buf[count + 1] = *info >> 8;
445 }
446}
447
448static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
449 u64 *correct_bitmap)
450{
451 struct mtd_info *mtd = nand_to_mtd(nand);
452 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
453 __le64 *info;
454 int ret = 0, i;
455
456 for (i = 0; i < nand->ecc.steps; i++) {
457 info = &meson_chip->info_buf[i];
458 if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
459 mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
460 *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
461 *correct_bitmap |= BIT_ULL(i);
462 continue;
463 }
464 if ((nand->options & NAND_NEED_SCRAMBLING) &&
465 ECC_ZERO_CNT(*info) < nand->ecc.strength) {
466 mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
467 *bitflips = max_t(u32, *bitflips,
468 ECC_ZERO_CNT(*info));
469 ret = ECC_CHECK_RETURN_FF;
470 } else {
471 ret = -EBADMSG;
472 }
473 }
474 return ret;
475}
476
477static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
478 int datalen, void *infobuf, int infolen,
479 enum dma_data_direction dir)
480{
481 struct meson_nfc *nfc = nand_get_controller_data(nand);
482 u32 cmd;
483 int ret = 0;
484
485 nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
486 ret = dma_mapping_error(nfc->dev, nfc->daddr);
487 if (ret) {
488 dev_err(nfc->dev, "DMA mapping error\n");
489 return ret;
490 }
491 cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
492 writel(cmd, nfc->reg_base + NFC_REG_CMD);
493
494 cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
495 writel(cmd, nfc->reg_base + NFC_REG_CMD);
496
497 if (infobuf) {
498 nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
499 ret = dma_mapping_error(nfc->dev, nfc->iaddr);
500 if (ret) {
501 dev_err(nfc->dev, "DMA mapping error\n");
502 dma_unmap_single(nfc->dev,
503 nfc->daddr, datalen, dir);
504 return ret;
505 }
506 cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
507 writel(cmd, nfc->reg_base + NFC_REG_CMD);
508
509 cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
510 writel(cmd, nfc->reg_base + NFC_REG_CMD);
511 }
512
513 return ret;
514}
515
516static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
517 int datalen, int infolen,
518 enum dma_data_direction dir)
519{
520 struct meson_nfc *nfc = nand_get_controller_data(nand);
521
522 dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
523 if (infolen)
524 dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
525}
526
527static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
528{
529 struct meson_nfc *nfc = nand_get_controller_data(nand);
530 int ret = 0;
531 u32 cmd;
532 u8 *info;
533
534 info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
535 if (!info)
536 return -ENOMEM;
537
538 ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
539 PER_INFO_BYTE, DMA_FROM_DEVICE);
540 if (ret)
541 goto out;
542
543 cmd = NFC_CMD_N2M | (len & GENMASK(5, 0));
544 writel(cmd, nfc->reg_base + NFC_REG_CMD);
545
546 meson_nfc_drain_cmd(nfc);
547 meson_nfc_wait_cmd_finish(nfc, 1000);
548 meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
549
550out:
551 kfree(info);
552
553 return ret;
554}
555
556static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
557{
558 struct meson_nfc *nfc = nand_get_controller_data(nand);
559 int ret = 0;
560 u32 cmd;
561
562 ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
563 0, DMA_TO_DEVICE);
564 if (ret)
565 return ret;
566
567 cmd = NFC_CMD_M2N | (len & GENMASK(5, 0));
568 writel(cmd, nfc->reg_base + NFC_REG_CMD);
569
570 meson_nfc_drain_cmd(nfc);
571 meson_nfc_wait_cmd_finish(nfc, 1000);
572 meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
573
574 return ret;
575}
576
577static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
578 int page, bool in)
579{
580 const struct nand_sdr_timings *sdr =
581 nand_get_sdr_timings(nand_get_interface_config(nand));
582 struct mtd_info *mtd = nand_to_mtd(nand);
583 struct meson_nfc *nfc = nand_get_controller_data(nand);
584 u32 *addrs = nfc->cmdfifo.rw.addrs;
585 u32 cs = nfc->param.chip_select;
586 u32 cmd0, cmd_num, row_start;
587 int i;
588
589 cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
590
591 cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
592 nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
593
594 addrs[0] = cs | NFC_CMD_ALE | 0;
595 if (mtd->writesize <= 512) {
596 cmd_num--;
597 row_start = 1;
598 } else {
599 addrs[1] = cs | NFC_CMD_ALE | 0;
600 row_start = 2;
601 }
602
603 addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
604 addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
605
606 if (nand->options & NAND_ROW_ADDR_3)
607 addrs[row_start + 2] =
608 cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
609 else
610 cmd_num--;
611
612 /* subtract cmd1 */
613 cmd_num--;
614
615 for (i = 0; i < cmd_num; i++)
616 writel_relaxed(nfc->cmdfifo.cmd[i],
617 nfc->reg_base + NFC_REG_CMD);
618
619 if (in) {
620 nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
621 writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
622 meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max));
623 } else {
624 meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
625 }
626
627 return 0;
628}
629
630static int meson_nfc_write_page_sub(struct nand_chip *nand,
631 int page, int raw)
632{
633 const struct nand_sdr_timings *sdr =
634 nand_get_sdr_timings(nand_get_interface_config(nand));
635 struct mtd_info *mtd = nand_to_mtd(nand);
636 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
637 struct meson_nfc *nfc = nand_get_controller_data(nand);
638 int data_len, info_len;
639 u32 cmd;
640 int ret;
641
642 meson_nfc_select_chip(nand, nand->cur_cs);
643
644 data_len = mtd->writesize + mtd->oobsize;
645 info_len = nand->ecc.steps * PER_INFO_BYTE;
646
647 ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
648 if (ret)
649 return ret;
650
651 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
652 data_len, meson_chip->info_buf,
653 info_len, DMA_TO_DEVICE);
654 if (ret)
655 return ret;
656
657 if (nand->options & NAND_NEED_SCRAMBLING) {
658 meson_nfc_cmd_seed(nfc, page);
659 meson_nfc_cmd_access(nand, raw, DIRWRITE,
660 NFC_CMD_SCRAMBLER_ENABLE);
661 } else {
662 meson_nfc_cmd_access(nand, raw, DIRWRITE,
663 NFC_CMD_SCRAMBLER_DISABLE);
664 }
665
666 cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
667 writel(cmd, nfc->reg_base + NFC_REG_CMD);
668 meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max));
669
670 meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
671
672 return ret;
673}
674
675static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
676 int oob_required, int page)
677{
678 u8 *oob_buf = nand->oob_poi;
679
680 meson_nfc_set_data_oob(nand, buf, oob_buf);
681
682 return meson_nfc_write_page_sub(nand, page, 1);
683}
684
685static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
686 const u8 *buf, int oob_required, int page)
687{
688 struct mtd_info *mtd = nand_to_mtd(nand);
689 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
690 u8 *oob_buf = nand->oob_poi;
691
692 memcpy(meson_chip->data_buf, buf, mtd->writesize);
693 memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
694 meson_nfc_set_user_byte(nand, oob_buf);
695
696 return meson_nfc_write_page_sub(nand, page, 0);
697}
698
699static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
700 struct nand_chip *nand, int raw)
701{
702 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
703 __le64 *info;
704 u32 neccpages;
705 int ret;
706
707 neccpages = raw ? 1 : nand->ecc.steps;
708 info = &meson_chip->info_buf[neccpages - 1];
709 do {
710 usleep_range(10, 15);
711 /* info is updated by nfc dma engine*/
712 smp_rmb();
713 ret = *info & ECC_COMPLETE;
714 } while (!ret);
715}
716
717static int meson_nfc_read_page_sub(struct nand_chip *nand,
718 int page, int raw)
719{
720 struct mtd_info *mtd = nand_to_mtd(nand);
721 struct meson_nfc *nfc = nand_get_controller_data(nand);
722 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
723 int data_len, info_len;
724 int ret;
725
726 meson_nfc_select_chip(nand, nand->cur_cs);
727
728 data_len = mtd->writesize + mtd->oobsize;
729 info_len = nand->ecc.steps * PER_INFO_BYTE;
730
731 ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
732 if (ret)
733 return ret;
734
735 ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
736 data_len, meson_chip->info_buf,
737 info_len, DMA_FROM_DEVICE);
738 if (ret)
739 return ret;
740
741 if (nand->options & NAND_NEED_SCRAMBLING) {
742 meson_nfc_cmd_seed(nfc, page);
743 meson_nfc_cmd_access(nand, raw, DIRREAD,
744 NFC_CMD_SCRAMBLER_ENABLE);
745 } else {
746 meson_nfc_cmd_access(nand, raw, DIRREAD,
747 NFC_CMD_SCRAMBLER_DISABLE);
748 }
749
750 ret = meson_nfc_wait_dma_finish(nfc);
751 meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
752
753 meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
754
755 return ret;
756}
757
758static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
759 int oob_required, int page)
760{
761 u8 *oob_buf = nand->oob_poi;
762 int ret;
763
764 ret = meson_nfc_read_page_sub(nand, page, 1);
765 if (ret)
766 return ret;
767
768 meson_nfc_get_data_oob(nand, buf, oob_buf);
769
770 return 0;
771}
772
773static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
774 int oob_required, int page)
775{
776 struct mtd_info *mtd = nand_to_mtd(nand);
777 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
778 struct nand_ecc_ctrl *ecc = &nand->ecc;
779 u64 correct_bitmap = 0;
780 u32 bitflips = 0;
781 u8 *oob_buf = nand->oob_poi;
782 int ret, i;
783
784 ret = meson_nfc_read_page_sub(nand, page, 0);
785 if (ret)
786 return ret;
787
788 meson_nfc_get_user_byte(nand, oob_buf);
789 ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
790 if (ret == ECC_CHECK_RETURN_FF) {
791 if (buf)
792 memset(buf, 0xff, mtd->writesize);
793 memset(oob_buf, 0xff, mtd->oobsize);
794 } else if (ret < 0) {
795 if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
796 mtd->ecc_stats.failed++;
797 return bitflips;
798 }
799 ret = meson_nfc_read_page_raw(nand, buf, 0, page);
800 if (ret)
801 return ret;
802
803 for (i = 0; i < nand->ecc.steps ; i++) {
804 u8 *data = buf + i * ecc->size;
805 u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
806
807 if (correct_bitmap & BIT_ULL(i))
808 continue;
809 ret = nand_check_erased_ecc_chunk(data, ecc->size,
810 oob, ecc->bytes + 2,
811 NULL, 0,
812 ecc->strength);
813 if (ret < 0) {
814 mtd->ecc_stats.failed++;
815 } else {
816 mtd->ecc_stats.corrected += ret;
817 bitflips = max_t(u32, bitflips, ret);
818 }
819 }
820 } else if (buf && buf != meson_chip->data_buf) {
821 memcpy(buf, meson_chip->data_buf, mtd->writesize);
822 }
823
824 return bitflips;
825}
826
827static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
828{
829 return meson_nfc_read_page_raw(nand, NULL, 1, page);
830}
831
832static int meson_nfc_read_oob(struct nand_chip *nand, int page)
833{
834 return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
835}
836
837static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
838{
839 if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
840 return true;
841 return false;
842}
843
844static void *
845meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
846{
847 if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
848 return NULL;
849
850 if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
851 return instr->ctx.data.buf.in;
852
853 return kzalloc(instr->ctx.data.len, GFP_KERNEL);
854}
855
856static void
857meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
858 void *buf)
859{
860 if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
861 WARN_ON(!buf))
862 return;
863
864 if (buf == instr->ctx.data.buf.in)
865 return;
866
867 memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
868 kfree(buf);
869}
870
871static void *
872meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
873{
874 if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
875 return NULL;
876
877 if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
878 return (void *)instr->ctx.data.buf.out;
879
880 return kmemdup(instr->ctx.data.buf.out,
881 instr->ctx.data.len, GFP_KERNEL);
882}
883
884static void
885meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
886 const void *buf)
887{
888 if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
889 WARN_ON(!buf))
890 return;
891
892 if (buf != instr->ctx.data.buf.out)
893 kfree(buf);
894}
895
896static int meson_nfc_exec_op(struct nand_chip *nand,
897 const struct nand_operation *op, bool check_only)
898{
899 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
900 struct meson_nfc *nfc = nand_get_controller_data(nand);
901 const struct nand_op_instr *instr = NULL;
902 void *buf;
903 u32 op_id, delay_idle, cmd;
904 int i;
905
906 if (check_only)
907 return 0;
908
909 meson_nfc_select_chip(nand, op->cs);
910 for (op_id = 0; op_id < op->ninstrs; op_id++) {
911 instr = &op->instrs[op_id];
912 delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
913 meson_chip->level1_divider *
914 NFC_CLK_CYCLE);
915 switch (instr->type) {
916 case NAND_OP_CMD_INSTR:
917 cmd = nfc->param.chip_select | NFC_CMD_CLE;
918 cmd |= instr->ctx.cmd.opcode & 0xff;
919 writel(cmd, nfc->reg_base + NFC_REG_CMD);
920 meson_nfc_cmd_idle(nfc, delay_idle);
921 break;
922
923 case NAND_OP_ADDR_INSTR:
924 for (i = 0; i < instr->ctx.addr.naddrs; i++) {
925 cmd = nfc->param.chip_select | NFC_CMD_ALE;
926 cmd |= instr->ctx.addr.addrs[i] & 0xff;
927 writel(cmd, nfc->reg_base + NFC_REG_CMD);
928 }
929 meson_nfc_cmd_idle(nfc, delay_idle);
930 break;
931
932 case NAND_OP_DATA_IN_INSTR:
933 buf = meson_nand_op_get_dma_safe_input_buf(instr);
934 if (!buf)
935 return -ENOMEM;
936 meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
937 meson_nand_op_put_dma_safe_input_buf(instr, buf);
938 break;
939
940 case NAND_OP_DATA_OUT_INSTR:
941 buf = meson_nand_op_get_dma_safe_output_buf(instr);
942 if (!buf)
943 return -ENOMEM;
944 meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
945 meson_nand_op_put_dma_safe_output_buf(instr, buf);
946 break;
947
948 case NAND_OP_WAITRDY_INSTR:
949 meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms);
950 if (instr->delay_ns)
951 meson_nfc_cmd_idle(nfc, delay_idle);
952 break;
953 }
954 }
955 meson_nfc_wait_cmd_finish(nfc, 1000);
956 return 0;
957}
958
959static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
960 struct mtd_oob_region *oobregion)
961{
962 struct nand_chip *nand = mtd_to_nand(mtd);
963
964 if (section >= nand->ecc.steps)
965 return -ERANGE;
966
967 oobregion->offset = 2 + (section * (2 + nand->ecc.bytes));
968 oobregion->length = nand->ecc.bytes;
969
970 return 0;
971}
972
973static int meson_ooblayout_free(struct mtd_info *mtd, int section,
974 struct mtd_oob_region *oobregion)
975{
976 struct nand_chip *nand = mtd_to_nand(mtd);
977
978 if (section >= nand->ecc.steps)
979 return -ERANGE;
980
981 oobregion->offset = section * (2 + nand->ecc.bytes);
982 oobregion->length = 2;
983
984 return 0;
985}
986
987static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
988 .ecc = meson_ooblayout_ecc,
989 .free = meson_ooblayout_free,
990};
991
992static int meson_nfc_clk_init(struct meson_nfc *nfc)
993{
994 struct clk_parent_data nfc_divider_parent_data[1];
995 struct clk_init_data init = {0};
996 int ret;
997
998 /* request core clock */
999 nfc->core_clk = devm_clk_get(nfc->dev, "core");
1000 if (IS_ERR(nfc->core_clk)) {
1001 dev_err(nfc->dev, "failed to get core clock\n");
1002 return PTR_ERR(nfc->core_clk);
1003 }
1004
1005 nfc->device_clk = devm_clk_get(nfc->dev, "device");
1006 if (IS_ERR(nfc->device_clk)) {
1007 dev_err(nfc->dev, "failed to get device clock\n");
1008 return PTR_ERR(nfc->device_clk);
1009 }
1010
1011 init.name = devm_kasprintf(nfc->dev,
1012 GFP_KERNEL, "%s#div",
1013 dev_name(nfc->dev));
1014 init.ops = &clk_divider_ops;
1015 nfc_divider_parent_data[0].fw_name = "device";
1016 init.parent_data = nfc_divider_parent_data;
1017 init.num_parents = 1;
1018 nfc->nand_divider.reg = nfc->reg_clk;
1019 nfc->nand_divider.shift = CLK_DIV_SHIFT;
1020 nfc->nand_divider.width = CLK_DIV_WIDTH;
1021 nfc->nand_divider.hw.init = &init;
1022 nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1023 CLK_DIVIDER_ROUND_CLOSEST |
1024 CLK_DIVIDER_ALLOW_ZERO;
1025
1026 nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw);
1027 if (IS_ERR(nfc->nand_clk))
1028 return PTR_ERR(nfc->nand_clk);
1029
1030 /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1031 writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
1032 nfc->reg_clk);
1033
1034 ret = clk_prepare_enable(nfc->core_clk);
1035 if (ret) {
1036 dev_err(nfc->dev, "failed to enable core clock\n");
1037 return ret;
1038 }
1039
1040 ret = clk_prepare_enable(nfc->device_clk);
1041 if (ret) {
1042 dev_err(nfc->dev, "failed to enable device clock\n");
1043 goto err_device_clk;
1044 }
1045
1046 ret = clk_prepare_enable(nfc->nand_clk);
1047 if (ret) {
1048 dev_err(nfc->dev, "pre enable NFC divider fail\n");
1049 goto err_nand_clk;
1050 }
1051
1052 ret = clk_set_rate(nfc->nand_clk, 24000000);
1053 if (ret)
1054 goto err_disable_clk;
1055
1056 return 0;
1057
1058err_disable_clk:
1059 clk_disable_unprepare(nfc->nand_clk);
1060err_nand_clk:
1061 clk_disable_unprepare(nfc->device_clk);
1062err_device_clk:
1063 clk_disable_unprepare(nfc->core_clk);
1064 return ret;
1065}
1066
1067static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1068{
1069 clk_disable_unprepare(nfc->nand_clk);
1070 clk_disable_unprepare(nfc->device_clk);
1071 clk_disable_unprepare(nfc->core_clk);
1072}
1073
1074static void meson_nfc_free_buffer(struct nand_chip *nand)
1075{
1076 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1077
1078 kfree(meson_chip->info_buf);
1079 kfree(meson_chip->data_buf);
1080}
1081
1082static int meson_chip_buffer_init(struct nand_chip *nand)
1083{
1084 struct mtd_info *mtd = nand_to_mtd(nand);
1085 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1086 u32 page_bytes, info_bytes, nsectors;
1087
1088 nsectors = mtd->writesize / nand->ecc.size;
1089
1090 page_bytes = mtd->writesize + mtd->oobsize;
1091 info_bytes = nsectors * PER_INFO_BYTE;
1092
1093 meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1094 if (!meson_chip->data_buf)
1095 return -ENOMEM;
1096
1097 meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1098 if (!meson_chip->info_buf) {
1099 kfree(meson_chip->data_buf);
1100 return -ENOMEM;
1101 }
1102
1103 return 0;
1104}
1105
1106static
1107int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1108 const struct nand_interface_config *conf)
1109{
1110 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1111 const struct nand_sdr_timings *timings;
1112 u32 div, bt_min, bt_max, tbers_clocks;
1113
1114 timings = nand_get_sdr_timings(conf);
1115 if (IS_ERR(timings))
1116 return -ENOTSUPP;
1117
1118 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1119 return 0;
1120
1121 div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1122 bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1123 bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1124 timings->tRC_min / 2) / div;
1125
1126 meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1127 div * NFC_CLK_CYCLE);
1128 meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1129 div * NFC_CLK_CYCLE);
1130 tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1131 div * NFC_CLK_CYCLE);
1132 meson_chip->tbers_max = ilog2(tbers_clocks);
1133 if (!is_power_of_2(tbers_clocks))
1134 meson_chip->tbers_max++;
1135
1136 bt_min = DIV_ROUND_UP(bt_min, 1000);
1137 bt_max = DIV_ROUND_UP(bt_max, 1000);
1138
1139 if (bt_max < bt_min)
1140 return -EINVAL;
1141
1142 meson_chip->level1_divider = div;
1143 meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1144 meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1145
1146 return 0;
1147}
1148
1149static int meson_nand_bch_mode(struct nand_chip *nand)
1150{
1151 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1152 int i;
1153
1154 if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1155 return -EINVAL;
1156
1157 for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1158 if (meson_ecc[i].strength == nand->ecc.strength) {
1159 meson_chip->bch_mode = meson_ecc[i].bch;
1160 return 0;
1161 }
1162 }
1163
1164 return -EINVAL;
1165}
1166
1167static void meson_nand_detach_chip(struct nand_chip *nand)
1168{
1169 meson_nfc_free_buffer(nand);
1170}
1171
1172static int meson_nand_attach_chip(struct nand_chip *nand)
1173{
1174 struct meson_nfc *nfc = nand_get_controller_data(nand);
1175 struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1176 struct mtd_info *mtd = nand_to_mtd(nand);
1177 int nsectors = mtd->writesize / 1024;
1178 int ret;
1179
1180 if (!mtd->name) {
1181 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1182 "%s:nand%d",
1183 dev_name(nfc->dev),
1184 meson_chip->sels[0]);
1185 if (!mtd->name)
1186 return -ENOMEM;
1187 }
1188
1189 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1190 nand->bbt_options |= NAND_BBT_NO_OOB;
1191
1192 nand->options |= NAND_NO_SUBPAGE_WRITE;
1193
1194 ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1195 mtd->oobsize - 2 * nsectors);
1196 if (ret) {
1197 dev_err(nfc->dev, "failed to ECC init\n");
1198 return -EINVAL;
1199 }
1200
1201 mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1202
1203 ret = meson_nand_bch_mode(nand);
1204 if (ret)
1205 return -EINVAL;
1206
1207 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1208 nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1209 nand->ecc.write_page = meson_nfc_write_page_hwecc;
1210 nand->ecc.write_oob_raw = nand_write_oob_std;
1211 nand->ecc.write_oob = nand_write_oob_std;
1212
1213 nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1214 nand->ecc.read_page = meson_nfc_read_page_hwecc;
1215 nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1216 nand->ecc.read_oob = meson_nfc_read_oob;
1217
1218 if (nand->options & NAND_BUSWIDTH_16) {
1219 dev_err(nfc->dev, "16bits bus width not supported");
1220 return -EINVAL;
1221 }
1222 ret = meson_chip_buffer_init(nand);
1223 if (ret)
1224 return -ENOMEM;
1225
1226 return ret;
1227}
1228
1229static const struct nand_controller_ops meson_nand_controller_ops = {
1230 .attach_chip = meson_nand_attach_chip,
1231 .detach_chip = meson_nand_detach_chip,
1232 .setup_interface = meson_nfc_setup_interface,
1233 .exec_op = meson_nfc_exec_op,
1234};
1235
1236static int
1237meson_nfc_nand_chip_init(struct device *dev,
1238 struct meson_nfc *nfc, struct device_node *np)
1239{
1240 struct meson_nfc_nand_chip *meson_chip;
1241 struct nand_chip *nand;
1242 struct mtd_info *mtd;
1243 int ret, i;
1244 u32 tmp, nsels;
1245
1246 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1247 if (!nsels || nsels > MAX_CE_NUM) {
1248 dev_err(dev, "invalid register property size\n");
1249 return -EINVAL;
1250 }
1251
1252 meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1253 GFP_KERNEL);
1254 if (!meson_chip)
1255 return -ENOMEM;
1256
1257 meson_chip->nsels = nsels;
1258
1259 for (i = 0; i < nsels; i++) {
1260 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1261 if (ret) {
1262 dev_err(dev, "could not retrieve register property: %d\n",
1263 ret);
1264 return ret;
1265 }
1266
1267 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1268 dev_err(dev, "CS %d already assigned\n", tmp);
1269 return -EINVAL;
1270 }
1271 }
1272
1273 nand = &meson_chip->nand;
1274 nand->controller = &nfc->controller;
1275 nand->controller->ops = &meson_nand_controller_ops;
1276 nand_set_flash_node(nand, np);
1277 nand_set_controller_data(nand, nfc);
1278
1279 nand->options |= NAND_USES_DMA;
1280 mtd = nand_to_mtd(nand);
1281 mtd->owner = THIS_MODULE;
1282 mtd->dev.parent = dev;
1283
1284 ret = nand_scan(nand, nsels);
1285 if (ret)
1286 return ret;
1287
1288 ret = mtd_device_register(mtd, NULL, 0);
1289 if (ret) {
1290 dev_err(dev, "failed to register MTD device: %d\n", ret);
1291 nand_cleanup(nand);
1292 return ret;
1293 }
1294
1295 list_add_tail(&meson_chip->node, &nfc->chips);
1296
1297 return 0;
1298}
1299
1300static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1301{
1302 struct meson_nfc_nand_chip *meson_chip;
1303 struct mtd_info *mtd;
1304
1305 while (!list_empty(&nfc->chips)) {
1306 meson_chip = list_first_entry(&nfc->chips,
1307 struct meson_nfc_nand_chip, node);
1308 mtd = nand_to_mtd(&meson_chip->nand);
1309 WARN_ON(mtd_device_unregister(mtd));
1310
1311 nand_cleanup(&meson_chip->nand);
1312 list_del(&meson_chip->node);
1313 }
1314}
1315
1316static int meson_nfc_nand_chips_init(struct device *dev,
1317 struct meson_nfc *nfc)
1318{
1319 struct device_node *np = dev->of_node;
1320 struct device_node *nand_np;
1321 int ret;
1322
1323 for_each_child_of_node(np, nand_np) {
1324 ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1325 if (ret) {
1326 meson_nfc_nand_chip_cleanup(nfc);
1327 of_node_put(nand_np);
1328 return ret;
1329 }
1330 }
1331
1332 return 0;
1333}
1334
1335static irqreturn_t meson_nfc_irq(int irq, void *id)
1336{
1337 struct meson_nfc *nfc = id;
1338 u32 cfg;
1339
1340 cfg = readl(nfc->reg_base + NFC_REG_CFG);
1341 if (!(cfg & NFC_RB_IRQ_EN))
1342 return IRQ_NONE;
1343
1344 cfg &= ~(NFC_RB_IRQ_EN);
1345 writel(cfg, nfc->reg_base + NFC_REG_CFG);
1346
1347 complete(&nfc->completion);
1348 return IRQ_HANDLED;
1349}
1350
1351static const struct meson_nfc_data meson_gxl_data = {
1352 .ecc_caps = &meson_gxl_ecc_caps,
1353};
1354
1355static const struct meson_nfc_data meson_axg_data = {
1356 .ecc_caps = &meson_axg_ecc_caps,
1357};
1358
1359static const struct of_device_id meson_nfc_id_table[] = {
1360 {
1361 .compatible = "amlogic,meson-gxl-nfc",
1362 .data = &meson_gxl_data,
1363 }, {
1364 .compatible = "amlogic,meson-axg-nfc",
1365 .data = &meson_axg_data,
1366 },
1367 {}
1368};
1369MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1370
1371static int meson_nfc_probe(struct platform_device *pdev)
1372{
1373 struct device *dev = &pdev->dev;
1374 struct meson_nfc *nfc;
1375 int ret, irq;
1376
1377 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1378 if (!nfc)
1379 return -ENOMEM;
1380
1381 nfc->data = of_device_get_match_data(&pdev->dev);
1382 if (!nfc->data)
1383 return -ENODEV;
1384
1385 nand_controller_init(&nfc->controller);
1386 INIT_LIST_HEAD(&nfc->chips);
1387 init_completion(&nfc->completion);
1388
1389 nfc->dev = dev;
1390
1391 nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc");
1392 if (IS_ERR(nfc->reg_base))
1393 return PTR_ERR(nfc->reg_base);
1394
1395 nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc");
1396 if (IS_ERR(nfc->reg_clk))
1397 return PTR_ERR(nfc->reg_clk);
1398
1399 irq = platform_get_irq(pdev, 0);
1400 if (irq < 0)
1401 return -EINVAL;
1402
1403 ret = meson_nfc_clk_init(nfc);
1404 if (ret) {
1405 dev_err(dev, "failed to initialize NAND clock\n");
1406 return ret;
1407 }
1408
1409 writel(0, nfc->reg_base + NFC_REG_CFG);
1410 ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1411 if (ret) {
1412 dev_err(dev, "failed to request NFC IRQ\n");
1413 ret = -EINVAL;
1414 goto err_clk;
1415 }
1416
1417 ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1418 if (ret) {
1419 dev_err(dev, "failed to set DMA mask\n");
1420 goto err_clk;
1421 }
1422
1423 platform_set_drvdata(pdev, nfc);
1424
1425 ret = meson_nfc_nand_chips_init(dev, nfc);
1426 if (ret) {
1427 dev_err(dev, "failed to init NAND chips\n");
1428 goto err_clk;
1429 }
1430
1431 return 0;
1432err_clk:
1433 meson_nfc_disable_clk(nfc);
1434 return ret;
1435}
1436
1437static int meson_nfc_remove(struct platform_device *pdev)
1438{
1439 struct meson_nfc *nfc = platform_get_drvdata(pdev);
1440
1441 meson_nfc_nand_chip_cleanup(nfc);
1442
1443 meson_nfc_disable_clk(nfc);
1444
1445 return 0;
1446}
1447
1448static struct platform_driver meson_nfc_driver = {
1449 .probe = meson_nfc_probe,
1450 .remove = meson_nfc_remove,
1451 .driver = {
1452 .name = "meson-nand",
1453 .of_match_table = meson_nfc_id_table,
1454 },
1455};
1456module_platform_driver(meson_nfc_driver);
1457
1458MODULE_LICENSE("Dual MIT/GPL");
1459MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1460MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");