Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Xilinx Versal memory controller driver
4 * Copyright (C) 2023 Advanced Micro Devices, Inc.
5 */
6#include <linux/bitfield.h>
7#include <linux/edac.h>
8#include <linux/interrupt.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_address.h>
12#include <linux/of_device.h>
13#include <linux/platform_device.h>
14#include <linux/sizes.h>
15#include <linux/firmware/xlnx-zynqmp.h>
16#include <linux/firmware/xlnx-event-manager.h>
17
18#include "edac_module.h"
19
20/* Granularity of reported error in bytes */
21#define XDDR_EDAC_ERR_GRAIN 1
22
23#define XDDR_EDAC_MSG_SIZE 256
24#define EVENT 2
25
26#define XDDR_PCSR_OFFSET 0xC
27#define XDDR_ISR_OFFSET 0x14
28#define XDDR_IRQ_EN_OFFSET 0x20
29#define XDDR_IRQ1_EN_OFFSET 0x2C
30#define XDDR_IRQ_DIS_OFFSET 0x24
31#define XDDR_IRQ_CE_MASK GENMASK(18, 15)
32#define XDDR_IRQ_UE_MASK GENMASK(14, 11)
33
34#define XDDR_REG_CONFIG0_OFFSET 0x258
35#define XDDR_REG_CONFIG0_BUS_WIDTH_MASK GENMASK(19, 18)
36#define XDDR_REG_CONFIG0_NUM_CHANS_MASK BIT(17)
37#define XDDR_REG_CONFIG0_NUM_RANKS_MASK GENMASK(15, 14)
38#define XDDR_REG_CONFIG0_SIZE_MASK GENMASK(10, 8)
39
40#define XDDR_REG_PINOUT_OFFSET 0x25C
41#define XDDR_REG_PINOUT_ECC_EN_MASK GENMASK(7, 5)
42
43#define ECCW0_FLIP_CTRL 0x109C
44#define ECCW0_FLIP0_OFFSET 0x10A0
45#define ECCW1_FLIP_CTRL 0x10AC
46#define ECCW1_FLIP0_OFFSET 0x10B0
47#define ECCR0_CERR_STAT_OFFSET 0x10BC
48#define ECCR0_CE_ADDR_LO_OFFSET 0x10C0
49#define ECCR0_CE_ADDR_HI_OFFSET 0x10C4
50#define ECCR0_CE_DATA_LO_OFFSET 0x10C8
51#define ECCR0_CE_DATA_HI_OFFSET 0x10CC
52#define ECCR0_CE_DATA_PAR_OFFSET 0x10D0
53
54#define ECCR0_UERR_STAT_OFFSET 0x10D4
55#define ECCR0_UE_ADDR_LO_OFFSET 0x10D8
56#define ECCR0_UE_ADDR_HI_OFFSET 0x10DC
57#define ECCR0_UE_DATA_LO_OFFSET 0x10E0
58#define ECCR0_UE_DATA_HI_OFFSET 0x10E4
59#define ECCR0_UE_DATA_PAR_OFFSET 0x10E8
60
61#define ECCR1_CERR_STAT_OFFSET 0x10F4
62#define ECCR1_CE_ADDR_LO_OFFSET 0x10F8
63#define ECCR1_CE_ADDR_HI_OFFSET 0x10FC
64#define ECCR1_CE_DATA_LO_OFFSET 0x1100
65#define ECCR1_CE_DATA_HI_OFFSET 0x110C
66#define ECCR1_CE_DATA_PAR_OFFSET 0x1108
67
68#define ECCR1_UERR_STAT_OFFSET 0x110C
69#define ECCR1_UE_ADDR_LO_OFFSET 0x1110
70#define ECCR1_UE_ADDR_HI_OFFSET 0x1114
71#define ECCR1_UE_DATA_LO_OFFSET 0x1118
72#define ECCR1_UE_DATA_HI_OFFSET 0x111C
73#define ECCR1_UE_DATA_PAR_OFFSET 0x1120
74
75#define XDDR_NOC_REG_ADEC4_OFFSET 0x44
76#define RANK_1_MASK GENMASK(11, 6)
77#define LRANK_0_MASK GENMASK(17, 12)
78#define LRANK_1_MASK GENMASK(23, 18)
79#define MASK_24 GENMASK(29, 24)
80
81#define XDDR_NOC_REG_ADEC5_OFFSET 0x48
82#define XDDR_NOC_REG_ADEC6_OFFSET 0x4C
83#define XDDR_NOC_REG_ADEC7_OFFSET 0x50
84#define XDDR_NOC_REG_ADEC8_OFFSET 0x54
85#define XDDR_NOC_REG_ADEC9_OFFSET 0x58
86#define XDDR_NOC_REG_ADEC10_OFFSET 0x5C
87
88#define XDDR_NOC_REG_ADEC11_OFFSET 0x60
89#define MASK_0 GENMASK(5, 0)
90#define GRP_0_MASK GENMASK(11, 6)
91#define GRP_1_MASK GENMASK(17, 12)
92#define CH_0_MASK GENMASK(23, 18)
93
94#define XDDR_NOC_REG_ADEC12_OFFSET 0x71C
95#define XDDR_NOC_REG_ADEC13_OFFSET 0x720
96
97#define XDDR_NOC_REG_ADEC14_OFFSET 0x724
98#define XDDR_NOC_ROW_MATCH_MASK GENMASK(17, 0)
99#define XDDR_NOC_COL_MATCH_MASK GENMASK(27, 18)
100#define XDDR_NOC_BANK_MATCH_MASK GENMASK(29, 28)
101#define XDDR_NOC_GRP_MATCH_MASK GENMASK(31, 30)
102
103#define XDDR_NOC_REG_ADEC15_OFFSET 0x728
104#define XDDR_NOC_RANK_MATCH_MASK GENMASK(1, 0)
105#define XDDR_NOC_LRANK_MATCH_MASK GENMASK(4, 2)
106#define XDDR_NOC_CH_MATCH_MASK BIT(5)
107#define XDDR_NOC_MOD_SEL_MASK BIT(6)
108#define XDDR_NOC_MATCH_EN_MASK BIT(8)
109
110#define ECCR_UE_CE_ADDR_HI_ROW_MASK GENMASK(7, 0)
111
112#define XDDR_EDAC_NR_CSROWS 1
113#define XDDR_EDAC_NR_CHANS 1
114
115#define XDDR_BUS_WIDTH_64 0
116#define XDDR_BUS_WIDTH_32 1
117#define XDDR_BUS_WIDTH_16 2
118
119#define ECC_CEPOISON_MASK 0x1
120#define ECC_UEPOISON_MASK 0x3
121
122#define XDDR_MAX_ROW_CNT 18
123#define XDDR_MAX_COL_CNT 10
124#define XDDR_MAX_RANK_CNT 2
125#define XDDR_MAX_LRANK_CNT 3
126#define XDDR_MAX_BANK_CNT 2
127#define XDDR_MAX_GRP_CNT 2
128
129/*
130 * Config and system registers are usually locked. This is the
131 * code which unlocks them in order to accept writes. See
132 *
133 * https://docs.xilinx.com/r/en-US/am012-versal-register-reference/PCSR_LOCK-XRAM_SLCR-Register
134 */
135#define PCSR_UNLOCK_VAL 0xF9E8D7C6
136#define XDDR_ERR_TYPE_CE 0
137#define XDDR_ERR_TYPE_UE 1
138
139#define XILINX_DRAM_SIZE_4G 0
140#define XILINX_DRAM_SIZE_6G 1
141#define XILINX_DRAM_SIZE_8G 2
142#define XILINX_DRAM_SIZE_12G 3
143#define XILINX_DRAM_SIZE_16G 4
144#define XILINX_DRAM_SIZE_32G 5
145
146/**
147 * struct ecc_error_info - ECC error log information.
148 * @burstpos: Burst position.
149 * @lrank: Logical Rank number.
150 * @rank: Rank number.
151 * @group: Group number.
152 * @bank: Bank number.
153 * @col: Column number.
154 * @row: Row number.
155 * @rowhi: Row number higher bits.
156 * @i: ECC error info.
157 */
158union ecc_error_info {
159 struct {
160 u32 burstpos:3;
161 u32 lrank:3;
162 u32 rank:2;
163 u32 group:2;
164 u32 bank:2;
165 u32 col:10;
166 u32 row:10;
167 u32 rowhi;
168 };
169 u64 i;
170} __packed;
171
172union edac_info {
173 struct {
174 u32 row0:6;
175 u32 row1:6;
176 u32 row2:6;
177 u32 row3:6;
178 u32 row4:6;
179 u32 reserved:2;
180 };
181 struct {
182 u32 col1:6;
183 u32 col2:6;
184 u32 col3:6;
185 u32 col4:6;
186 u32 col5:6;
187 u32 reservedcol:2;
188 };
189 u32 i;
190} __packed;
191
192/**
193 * struct ecc_status - ECC status information to report.
194 * @ceinfo: Correctable error log information.
195 * @ueinfo: Uncorrectable error log information.
196 * @channel: Channel number.
197 * @error_type: Error type information.
198 */
199struct ecc_status {
200 union ecc_error_info ceinfo[2];
201 union ecc_error_info ueinfo[2];
202 u8 channel;
203 u8 error_type;
204};
205
206/**
207 * struct edac_priv - DDR memory controller private instance data.
208 * @ddrmc_baseaddr: Base address of the DDR controller.
209 * @ddrmc_noc_baseaddr: Base address of the DDRMC NOC.
210 * @message: Buffer for framing the event specific info.
211 * @mc_id: Memory controller ID.
212 * @ce_cnt: Correctable error count.
213 * @ue_cnt: UnCorrectable error count.
214 * @stat: ECC status information.
215 * @lrank_bit: Bit shifts for lrank bit.
216 * @rank_bit: Bit shifts for rank bit.
217 * @row_bit: Bit shifts for row bit.
218 * @col_bit: Bit shifts for column bit.
219 * @bank_bit: Bit shifts for bank bit.
220 * @grp_bit: Bit shifts for group bit.
221 * @ch_bit: Bit shifts for channel bit.
222 * @err_inject_addr: Data poison address.
223 * @debugfs: Debugfs handle.
224 */
225struct edac_priv {
226 void __iomem *ddrmc_baseaddr;
227 void __iomem *ddrmc_noc_baseaddr;
228 char message[XDDR_EDAC_MSG_SIZE];
229 u32 mc_id;
230 u32 ce_cnt;
231 u32 ue_cnt;
232 struct ecc_status stat;
233 u32 lrank_bit[3];
234 u32 rank_bit[2];
235 u32 row_bit[18];
236 u32 col_bit[10];
237 u32 bank_bit[2];
238 u32 grp_bit[2];
239 u32 ch_bit;
240#ifdef CONFIG_EDAC_DEBUG
241 u64 err_inject_addr;
242 struct dentry *debugfs;
243#endif
244};
245
246static void get_ce_error_info(struct edac_priv *priv)
247{
248 void __iomem *ddrmc_base;
249 struct ecc_status *p;
250 u32 regval;
251 u64 reghi;
252
253 ddrmc_base = priv->ddrmc_baseaddr;
254 p = &priv->stat;
255
256 p->error_type = XDDR_ERR_TYPE_CE;
257 regval = readl(ddrmc_base + ECCR0_CE_ADDR_LO_OFFSET);
258 reghi = regval & ECCR_UE_CE_ADDR_HI_ROW_MASK;
259 p->ceinfo[0].i = regval | reghi << 32;
260 regval = readl(ddrmc_base + ECCR0_CE_ADDR_HI_OFFSET);
261
262 edac_dbg(2, "ERR DATA: 0x%08X%08X ERR DATA PARITY: 0x%08X\n",
263 readl(ddrmc_base + ECCR0_CE_DATA_LO_OFFSET),
264 readl(ddrmc_base + ECCR0_CE_DATA_HI_OFFSET),
265 readl(ddrmc_base + ECCR0_CE_DATA_PAR_OFFSET));
266
267 regval = readl(ddrmc_base + ECCR1_CE_ADDR_LO_OFFSET);
268 reghi = readl(ddrmc_base + ECCR1_CE_ADDR_HI_OFFSET);
269 p->ceinfo[1].i = regval | reghi << 32;
270 regval = readl(ddrmc_base + ECCR1_CE_ADDR_HI_OFFSET);
271
272 edac_dbg(2, "ERR DATA: 0x%08X%08X ERR DATA PARITY: 0x%08X\n",
273 readl(ddrmc_base + ECCR1_CE_DATA_LO_OFFSET),
274 readl(ddrmc_base + ECCR1_CE_DATA_HI_OFFSET),
275 readl(ddrmc_base + ECCR1_CE_DATA_PAR_OFFSET));
276}
277
278static void get_ue_error_info(struct edac_priv *priv)
279{
280 void __iomem *ddrmc_base;
281 struct ecc_status *p;
282 u32 regval;
283 u64 reghi;
284
285 ddrmc_base = priv->ddrmc_baseaddr;
286 p = &priv->stat;
287
288 p->error_type = XDDR_ERR_TYPE_UE;
289 regval = readl(ddrmc_base + ECCR0_UE_ADDR_LO_OFFSET);
290 reghi = readl(ddrmc_base + ECCR0_UE_ADDR_HI_OFFSET);
291
292 p->ueinfo[0].i = regval | reghi << 32;
293 regval = readl(ddrmc_base + ECCR0_UE_ADDR_HI_OFFSET);
294
295 edac_dbg(2, "ERR DATA: 0x%08X%08X ERR DATA PARITY: 0x%08X\n",
296 readl(ddrmc_base + ECCR0_UE_DATA_LO_OFFSET),
297 readl(ddrmc_base + ECCR0_UE_DATA_HI_OFFSET),
298 readl(ddrmc_base + ECCR0_UE_DATA_PAR_OFFSET));
299
300 regval = readl(ddrmc_base + ECCR1_UE_ADDR_LO_OFFSET);
301 reghi = readl(ddrmc_base + ECCR1_UE_ADDR_HI_OFFSET);
302 p->ueinfo[1].i = regval | reghi << 32;
303
304 edac_dbg(2, "ERR DATA: 0x%08X%08X ERR DATA PARITY: 0x%08X\n",
305 readl(ddrmc_base + ECCR1_UE_DATA_LO_OFFSET),
306 readl(ddrmc_base + ECCR1_UE_DATA_HI_OFFSET),
307 readl(ddrmc_base + ECCR1_UE_DATA_PAR_OFFSET));
308}
309
310static bool get_error_info(struct edac_priv *priv)
311{
312 u32 eccr0_ceval, eccr1_ceval, eccr0_ueval, eccr1_ueval;
313 void __iomem *ddrmc_base;
314 struct ecc_status *p;
315
316 ddrmc_base = priv->ddrmc_baseaddr;
317 p = &priv->stat;
318
319 eccr0_ceval = readl(ddrmc_base + ECCR0_CERR_STAT_OFFSET);
320 eccr1_ceval = readl(ddrmc_base + ECCR1_CERR_STAT_OFFSET);
321 eccr0_ueval = readl(ddrmc_base + ECCR0_UERR_STAT_OFFSET);
322 eccr1_ueval = readl(ddrmc_base + ECCR1_UERR_STAT_OFFSET);
323
324 if (!eccr0_ceval && !eccr1_ceval && !eccr0_ueval && !eccr1_ueval)
325 return 1;
326 if (!eccr0_ceval)
327 p->channel = 1;
328 else
329 p->channel = 0;
330
331 if (eccr0_ceval || eccr1_ceval)
332 get_ce_error_info(priv);
333
334 if (eccr0_ueval || eccr1_ueval) {
335 if (!eccr0_ueval)
336 p->channel = 1;
337 else
338 p->channel = 0;
339 get_ue_error_info(priv);
340 }
341
342 /* Unlock the PCSR registers */
343 writel(PCSR_UNLOCK_VAL, ddrmc_base + XDDR_PCSR_OFFSET);
344
345 writel(0, ddrmc_base + ECCR0_CERR_STAT_OFFSET);
346 writel(0, ddrmc_base + ECCR1_CERR_STAT_OFFSET);
347 writel(0, ddrmc_base + ECCR0_UERR_STAT_OFFSET);
348 writel(0, ddrmc_base + ECCR1_UERR_STAT_OFFSET);
349
350 /* Lock the PCSR registers */
351 writel(1, ddrmc_base + XDDR_PCSR_OFFSET);
352
353 return 0;
354}
355
356/**
357 * convert_to_physical - Convert to physical address.
358 * @priv: DDR memory controller private instance data.
359 * @pinf: ECC error info structure.
360 *
361 * Return: Physical address of the DDR memory.
362 */
363static unsigned long convert_to_physical(struct edac_priv *priv, union ecc_error_info pinf)
364{
365 unsigned long err_addr = 0;
366 u32 index;
367 u32 row;
368
369 row = pinf.rowhi << 10 | pinf.row;
370 for (index = 0; index < XDDR_MAX_ROW_CNT; index++) {
371 err_addr |= (row & BIT(0)) << priv->row_bit[index];
372 row >>= 1;
373 }
374
375 for (index = 0; index < XDDR_MAX_COL_CNT; index++) {
376 err_addr |= (pinf.col & BIT(0)) << priv->col_bit[index];
377 pinf.col >>= 1;
378 }
379
380 for (index = 0; index < XDDR_MAX_BANK_CNT; index++) {
381 err_addr |= (pinf.bank & BIT(0)) << priv->bank_bit[index];
382 pinf.bank >>= 1;
383 }
384
385 for (index = 0; index < XDDR_MAX_GRP_CNT; index++) {
386 err_addr |= (pinf.group & BIT(0)) << priv->grp_bit[index];
387 pinf.group >>= 1;
388 }
389
390 for (index = 0; index < XDDR_MAX_RANK_CNT; index++) {
391 err_addr |= (pinf.rank & BIT(0)) << priv->rank_bit[index];
392 pinf.rank >>= 1;
393 }
394
395 for (index = 0; index < XDDR_MAX_LRANK_CNT; index++) {
396 err_addr |= (pinf.lrank & BIT(0)) << priv->lrank_bit[index];
397 pinf.lrank >>= 1;
398 }
399
400 err_addr |= (priv->stat.channel & BIT(0)) << priv->ch_bit;
401
402 return err_addr;
403}
404
405/**
406 * handle_error - Handle Correctable and Uncorrectable errors.
407 * @mci: EDAC memory controller instance.
408 * @stat: ECC status structure.
409 *
410 * Handles ECC correctable and uncorrectable errors.
411 */
412static void handle_error(struct mem_ctl_info *mci, struct ecc_status *stat)
413{
414 struct edac_priv *priv = mci->pvt_info;
415 union ecc_error_info pinf;
416
417 if (stat->error_type == XDDR_ERR_TYPE_CE) {
418 priv->ce_cnt++;
419 pinf = stat->ceinfo[stat->channel];
420 snprintf(priv->message, XDDR_EDAC_MSG_SIZE,
421 "Error type:%s MC ID: %d Addr at %lx Burst Pos: %d\n",
422 "CE", priv->mc_id,
423 convert_to_physical(priv, pinf), pinf.burstpos);
424
425 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci,
426 priv->ce_cnt, 0, 0, 0, 0, 0, -1,
427 priv->message, "");
428 }
429
430 if (stat->error_type == XDDR_ERR_TYPE_UE) {
431 priv->ue_cnt++;
432 pinf = stat->ueinfo[stat->channel];
433 snprintf(priv->message, XDDR_EDAC_MSG_SIZE,
434 "Error type:%s MC ID: %d Addr at %lx Burst Pos: %d\n",
435 "UE", priv->mc_id,
436 convert_to_physical(priv, pinf), pinf.burstpos);
437
438 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci,
439 priv->ue_cnt, 0, 0, 0, 0, 0, -1,
440 priv->message, "");
441 }
442
443 memset(stat, 0, sizeof(*stat));
444}
445
446/**
447 * err_callback - Handle Correctable and Uncorrectable errors.
448 * @payload: payload data.
449 * @data: mci controller data.
450 *
451 * Handles ECC correctable and uncorrectable errors.
452 */
453static void err_callback(const u32 *payload, void *data)
454{
455 struct mem_ctl_info *mci = (struct mem_ctl_info *)data;
456 struct edac_priv *priv;
457 struct ecc_status *p;
458 int regval;
459
460 priv = mci->pvt_info;
461 p = &priv->stat;
462
463 regval = readl(priv->ddrmc_baseaddr + XDDR_ISR_OFFSET);
464
465 if (payload[EVENT] == XPM_EVENT_ERROR_MASK_DDRMC_CR)
466 p->error_type = XDDR_ERR_TYPE_CE;
467 if (payload[EVENT] == XPM_EVENT_ERROR_MASK_DDRMC_NCR)
468 p->error_type = XDDR_ERR_TYPE_UE;
469
470 if (get_error_info(priv))
471 return;
472
473 handle_error(mci, &priv->stat);
474
475 /* Unlock the PCSR registers */
476 writel(PCSR_UNLOCK_VAL, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
477
478 /* Clear the ISR */
479 writel(regval, priv->ddrmc_baseaddr + XDDR_ISR_OFFSET);
480
481 /* Lock the PCSR registers */
482 writel(1, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
483 edac_dbg(3, "Total error count CE %d UE %d\n",
484 priv->ce_cnt, priv->ue_cnt);
485}
486
487/**
488 * get_dwidth - Return the controller memory width.
489 * @base: DDR memory controller base address.
490 *
491 * Get the EDAC device type width appropriate for the controller
492 * configuration.
493 *
494 * Return: a device type width enumeration.
495 */
496static enum dev_type get_dwidth(const void __iomem *base)
497{
498 enum dev_type dt;
499 u32 regval;
500 u32 width;
501
502 regval = readl(base + XDDR_REG_CONFIG0_OFFSET);
503 width = FIELD_GET(XDDR_REG_CONFIG0_BUS_WIDTH_MASK, regval);
504
505 switch (width) {
506 case XDDR_BUS_WIDTH_16:
507 dt = DEV_X2;
508 break;
509 case XDDR_BUS_WIDTH_32:
510 dt = DEV_X4;
511 break;
512 case XDDR_BUS_WIDTH_64:
513 dt = DEV_X8;
514 break;
515 default:
516 dt = DEV_UNKNOWN;
517 }
518
519 return dt;
520}
521
522/**
523 * get_ecc_state - Return the controller ECC enable/disable status.
524 * @base: DDR memory controller base address.
525 *
526 * Get the ECC enable/disable status for the controller.
527 *
528 * Return: a ECC status boolean i.e true/false - enabled/disabled.
529 */
530static bool get_ecc_state(void __iomem *base)
531{
532 enum dev_type dt;
533 u32 ecctype;
534
535 dt = get_dwidth(base);
536 if (dt == DEV_UNKNOWN)
537 return false;
538
539 ecctype = readl(base + XDDR_REG_PINOUT_OFFSET);
540 ecctype &= XDDR_REG_PINOUT_ECC_EN_MASK;
541
542 return !!ecctype;
543}
544
545/**
546 * get_memsize - Get the size of the attached memory device.
547 * @priv: DDR memory controller private instance data.
548 *
549 * Return: the memory size in bytes.
550 */
551static u64 get_memsize(struct edac_priv *priv)
552{
553 u32 regval;
554 u64 size;
555
556 regval = readl(priv->ddrmc_baseaddr + XDDR_REG_CONFIG0_OFFSET);
557 regval = FIELD_GET(XDDR_REG_CONFIG0_SIZE_MASK, regval);
558
559 switch (regval) {
560 case XILINX_DRAM_SIZE_4G:
561 size = 4U; break;
562 case XILINX_DRAM_SIZE_6G:
563 size = 6U; break;
564 case XILINX_DRAM_SIZE_8G:
565 size = 8U; break;
566 case XILINX_DRAM_SIZE_12G:
567 size = 12U; break;
568 case XILINX_DRAM_SIZE_16G:
569 size = 16U; break;
570 case XILINX_DRAM_SIZE_32G:
571 size = 32U; break;
572 /* Invalid configuration */
573 default:
574 size = 0; break;
575 }
576
577 size *= SZ_1G;
578 return size;
579}
580
581/**
582 * init_csrows - Initialize the csrow data.
583 * @mci: EDAC memory controller instance.
584 *
585 * Initialize the chip select rows associated with the EDAC memory
586 * controller instance.
587 */
588static void init_csrows(struct mem_ctl_info *mci)
589{
590 struct edac_priv *priv = mci->pvt_info;
591 struct csrow_info *csi;
592 struct dimm_info *dimm;
593 unsigned long size;
594 u32 row;
595 int ch;
596
597 size = get_memsize(priv);
598 for (row = 0; row < mci->nr_csrows; row++) {
599 csi = mci->csrows[row];
600 for (ch = 0; ch < csi->nr_channels; ch++) {
601 dimm = csi->channels[ch]->dimm;
602 dimm->edac_mode = EDAC_SECDED;
603 dimm->mtype = MEM_DDR4;
604 dimm->nr_pages = (size >> PAGE_SHIFT) / csi->nr_channels;
605 dimm->grain = XDDR_EDAC_ERR_GRAIN;
606 dimm->dtype = get_dwidth(priv->ddrmc_baseaddr);
607 }
608 }
609}
610
611/**
612 * mc_init - Initialize one driver instance.
613 * @mci: EDAC memory controller instance.
614 * @pdev: platform device.
615 *
616 * Perform initialization of the EDAC memory controller instance and
617 * related driver-private data associated with the memory controller the
618 * instance is bound to.
619 */
620static void mc_init(struct mem_ctl_info *mci, struct platform_device *pdev)
621{
622 mci->pdev = &pdev->dev;
623 platform_set_drvdata(pdev, mci);
624
625 /* Initialize controller capabilities and configuration */
626 mci->mtype_cap = MEM_FLAG_DDR4;
627 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
628 mci->scrub_cap = SCRUB_HW_SRC;
629 mci->scrub_mode = SCRUB_NONE;
630
631 mci->edac_cap = EDAC_FLAG_SECDED;
632 mci->ctl_name = "xlnx_ddr_controller";
633 mci->dev_name = dev_name(&pdev->dev);
634 mci->mod_name = "xlnx_edac";
635
636 edac_op_state = EDAC_OPSTATE_INT;
637
638 init_csrows(mci);
639}
640
641static void enable_intr(struct edac_priv *priv)
642{
643 /* Unlock the PCSR registers */
644 writel(PCSR_UNLOCK_VAL, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
645
646 /* Enable UE and CE Interrupts to support the interrupt case */
647 writel(XDDR_IRQ_CE_MASK | XDDR_IRQ_UE_MASK,
648 priv->ddrmc_baseaddr + XDDR_IRQ_EN_OFFSET);
649
650 writel(XDDR_IRQ_UE_MASK,
651 priv->ddrmc_baseaddr + XDDR_IRQ1_EN_OFFSET);
652 /* Lock the PCSR registers */
653 writel(1, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
654}
655
656static void disable_intr(struct edac_priv *priv)
657{
658 /* Unlock the PCSR registers */
659 writel(PCSR_UNLOCK_VAL, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
660
661 /* Disable UE/CE Interrupts */
662 writel(XDDR_IRQ_CE_MASK | XDDR_IRQ_UE_MASK,
663 priv->ddrmc_baseaddr + XDDR_IRQ_DIS_OFFSET);
664
665 /* Lock the PCSR registers */
666 writel(1, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
667}
668
669#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
670
671#ifdef CONFIG_EDAC_DEBUG
672/**
673 * poison_setup - Update poison registers.
674 * @priv: DDR memory controller private instance data.
675 *
676 * Update poison registers as per DDR mapping upon write of the address
677 * location the fault is injected.
678 * Return: none.
679 */
680static void poison_setup(struct edac_priv *priv)
681{
682 u32 col = 0, row = 0, bank = 0, grp = 0, rank = 0, lrank = 0, ch = 0;
683 u32 index, regval;
684
685 for (index = 0; index < XDDR_MAX_ROW_CNT; index++) {
686 row |= (((priv->err_inject_addr >> priv->row_bit[index]) &
687 BIT(0)) << index);
688 }
689
690 for (index = 0; index < XDDR_MAX_COL_CNT; index++) {
691 col |= (((priv->err_inject_addr >> priv->col_bit[index]) &
692 BIT(0)) << index);
693 }
694
695 for (index = 0; index < XDDR_MAX_BANK_CNT; index++) {
696 bank |= (((priv->err_inject_addr >> priv->bank_bit[index]) &
697 BIT(0)) << index);
698 }
699
700 for (index = 0; index < XDDR_MAX_GRP_CNT; index++) {
701 grp |= (((priv->err_inject_addr >> priv->grp_bit[index]) &
702 BIT(0)) << index);
703 }
704
705 for (index = 0; index < XDDR_MAX_RANK_CNT; index++) {
706 rank |= (((priv->err_inject_addr >> priv->rank_bit[index]) &
707 BIT(0)) << index);
708 }
709
710 for (index = 0; index < XDDR_MAX_LRANK_CNT; index++) {
711 lrank |= (((priv->err_inject_addr >> priv->lrank_bit[index]) &
712 BIT(0)) << index);
713 }
714
715 ch = (priv->err_inject_addr >> priv->ch_bit) & BIT(0);
716 if (ch)
717 writel(0xFF, priv->ddrmc_baseaddr + ECCW1_FLIP_CTRL);
718 else
719 writel(0xFF, priv->ddrmc_baseaddr + ECCW0_FLIP_CTRL);
720
721 writel(0, priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC12_OFFSET);
722 writel(0, priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC13_OFFSET);
723
724 regval = row & XDDR_NOC_ROW_MATCH_MASK;
725 regval |= FIELD_PREP(XDDR_NOC_COL_MATCH_MASK, col);
726 regval |= FIELD_PREP(XDDR_NOC_BANK_MATCH_MASK, bank);
727 regval |= FIELD_PREP(XDDR_NOC_GRP_MATCH_MASK, grp);
728 writel(regval, priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC14_OFFSET);
729
730 regval = rank & XDDR_NOC_RANK_MATCH_MASK;
731 regval |= FIELD_PREP(XDDR_NOC_LRANK_MATCH_MASK, lrank);
732 regval |= FIELD_PREP(XDDR_NOC_CH_MATCH_MASK, ch);
733 regval |= (XDDR_NOC_MOD_SEL_MASK | XDDR_NOC_MATCH_EN_MASK);
734 writel(regval, priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC15_OFFSET);
735}
736
737static ssize_t xddr_inject_data_poison_store(struct mem_ctl_info *mci,
738 const char __user *data)
739{
740 struct edac_priv *priv = mci->pvt_info;
741
742 writel(0, priv->ddrmc_baseaddr + ECCW0_FLIP0_OFFSET);
743 writel(0, priv->ddrmc_baseaddr + ECCW1_FLIP0_OFFSET);
744
745 if (strncmp(data, "CE", 2) == 0) {
746 writel(ECC_CEPOISON_MASK, priv->ddrmc_baseaddr +
747 ECCW0_FLIP0_OFFSET);
748 writel(ECC_CEPOISON_MASK, priv->ddrmc_baseaddr +
749 ECCW1_FLIP0_OFFSET);
750 } else {
751 writel(ECC_UEPOISON_MASK, priv->ddrmc_baseaddr +
752 ECCW0_FLIP0_OFFSET);
753 writel(ECC_UEPOISON_MASK, priv->ddrmc_baseaddr +
754 ECCW1_FLIP0_OFFSET);
755 }
756
757 /* Lock the PCSR registers */
758 writel(1, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
759
760 return 0;
761}
762
763static ssize_t inject_data_poison_store(struct file *file, const char __user *data,
764 size_t count, loff_t *ppos)
765{
766 struct device *dev = file->private_data;
767 struct mem_ctl_info *mci = to_mci(dev);
768 struct edac_priv *priv = mci->pvt_info;
769
770 /* Unlock the PCSR registers */
771 writel(PCSR_UNLOCK_VAL, priv->ddrmc_baseaddr + XDDR_PCSR_OFFSET);
772 writel(PCSR_UNLOCK_VAL, priv->ddrmc_noc_baseaddr + XDDR_PCSR_OFFSET);
773
774 poison_setup(priv);
775
776 /* Lock the PCSR registers */
777 writel(1, priv->ddrmc_noc_baseaddr + XDDR_PCSR_OFFSET);
778
779 xddr_inject_data_poison_store(mci, data);
780
781 return count;
782}
783
784static const struct file_operations xddr_inject_enable_fops = {
785 .open = simple_open,
786 .write = inject_data_poison_store,
787 .llseek = generic_file_llseek,
788};
789
790static void create_debugfs_attributes(struct mem_ctl_info *mci)
791{
792 struct edac_priv *priv = mci->pvt_info;
793
794 priv->debugfs = edac_debugfs_create_dir(mci->dev_name);
795 if (!priv->debugfs)
796 return;
797
798 edac_debugfs_create_file("inject_error", 0200, priv->debugfs,
799 &mci->dev, &xddr_inject_enable_fops);
800 debugfs_create_x64("address", 0600, priv->debugfs,
801 &priv->err_inject_addr);
802 mci->debugfs = priv->debugfs;
803}
804
805static inline void process_bit(struct edac_priv *priv, unsigned int start, u32 regval)
806{
807 union edac_info rows;
808
809 rows.i = regval;
810 priv->row_bit[start] = rows.row0;
811 priv->row_bit[start + 1] = rows.row1;
812 priv->row_bit[start + 2] = rows.row2;
813 priv->row_bit[start + 3] = rows.row3;
814 priv->row_bit[start + 4] = rows.row4;
815}
816
817static void setup_row_address_map(struct edac_priv *priv)
818{
819 u32 regval;
820 union edac_info rows;
821
822 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC5_OFFSET);
823 process_bit(priv, 0, regval);
824
825 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC6_OFFSET);
826 process_bit(priv, 5, regval);
827
828 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC7_OFFSET);
829 process_bit(priv, 10, regval);
830
831 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC8_OFFSET);
832 rows.i = regval;
833
834 priv->row_bit[15] = rows.row0;
835 priv->row_bit[16] = rows.row1;
836 priv->row_bit[17] = rows.row2;
837}
838
839static void setup_column_address_map(struct edac_priv *priv)
840{
841 u32 regval;
842 union edac_info cols;
843
844 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC8_OFFSET);
845 priv->col_bit[0] = FIELD_GET(MASK_24, regval);
846
847 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC9_OFFSET);
848 cols.i = regval;
849 priv->col_bit[1] = cols.col1;
850 priv->col_bit[2] = cols.col2;
851 priv->col_bit[3] = cols.col3;
852 priv->col_bit[4] = cols.col4;
853 priv->col_bit[5] = cols.col5;
854
855 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC10_OFFSET);
856 cols.i = regval;
857 priv->col_bit[6] = cols.col1;
858 priv->col_bit[7] = cols.col2;
859 priv->col_bit[8] = cols.col3;
860 priv->col_bit[9] = cols.col4;
861}
862
863static void setup_bank_grp_ch_address_map(struct edac_priv *priv)
864{
865 u32 regval;
866
867 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC10_OFFSET);
868 priv->bank_bit[0] = FIELD_GET(MASK_24, regval);
869
870 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC11_OFFSET);
871 priv->bank_bit[1] = (regval & MASK_0);
872 priv->grp_bit[0] = FIELD_GET(GRP_0_MASK, regval);
873 priv->grp_bit[1] = FIELD_GET(GRP_1_MASK, regval);
874 priv->ch_bit = FIELD_GET(CH_0_MASK, regval);
875}
876
877static void setup_rank_lrank_address_map(struct edac_priv *priv)
878{
879 u32 regval;
880
881 regval = readl(priv->ddrmc_noc_baseaddr + XDDR_NOC_REG_ADEC4_OFFSET);
882 priv->rank_bit[0] = (regval & MASK_0);
883 priv->rank_bit[1] = FIELD_GET(RANK_1_MASK, regval);
884 priv->lrank_bit[0] = FIELD_GET(LRANK_0_MASK, regval);
885 priv->lrank_bit[1] = FIELD_GET(LRANK_1_MASK, regval);
886 priv->lrank_bit[2] = FIELD_GET(MASK_24, regval);
887}
888
889/**
890 * setup_address_map - Set Address Map by querying ADDRMAP registers.
891 * @priv: DDR memory controller private instance data.
892 *
893 * Set Address Map by querying ADDRMAP registers.
894 *
895 * Return: none.
896 */
897static void setup_address_map(struct edac_priv *priv)
898{
899 setup_row_address_map(priv);
900
901 setup_column_address_map(priv);
902
903 setup_bank_grp_ch_address_map(priv);
904
905 setup_rank_lrank_address_map(priv);
906}
907#endif /* CONFIG_EDAC_DEBUG */
908
909static const struct of_device_id xlnx_edac_match[] = {
910 { .compatible = "xlnx,versal-ddrmc", },
911 {
912 /* end of table */
913 }
914};
915
916MODULE_DEVICE_TABLE(of, xlnx_edac_match);
917static u32 emif_get_id(struct device_node *node)
918{
919 u32 addr, my_addr, my_id = 0;
920 struct device_node *np;
921 const __be32 *addrp;
922
923 addrp = of_get_address(node, 0, NULL, NULL);
924 my_addr = (u32)of_translate_address(node, addrp);
925
926 for_each_matching_node(np, xlnx_edac_match) {
927 if (np == node)
928 continue;
929
930 addrp = of_get_address(np, 0, NULL, NULL);
931 addr = (u32)of_translate_address(np, addrp);
932
933 edac_printk(KERN_INFO, EDAC_MC,
934 "addr=%x, my_addr=%x\n",
935 addr, my_addr);
936
937 if (addr < my_addr)
938 my_id++;
939 }
940
941 return my_id;
942}
943
944static int mc_probe(struct platform_device *pdev)
945{
946 void __iomem *ddrmc_baseaddr, *ddrmc_noc_baseaddr;
947 struct edac_mc_layer layers[2];
948 struct mem_ctl_info *mci;
949 u8 num_chans, num_csrows;
950 struct edac_priv *priv;
951 u32 edac_mc_id, regval;
952 int rc;
953
954 ddrmc_baseaddr = devm_platform_ioremap_resource_byname(pdev, "base");
955 if (IS_ERR(ddrmc_baseaddr))
956 return PTR_ERR(ddrmc_baseaddr);
957
958 ddrmc_noc_baseaddr = devm_platform_ioremap_resource_byname(pdev, "noc");
959 if (IS_ERR(ddrmc_noc_baseaddr))
960 return PTR_ERR(ddrmc_noc_baseaddr);
961
962 if (!get_ecc_state(ddrmc_baseaddr))
963 return -ENXIO;
964
965 /* Allocate ID number for the EMIF controller */
966 edac_mc_id = emif_get_id(pdev->dev.of_node);
967
968 regval = readl(ddrmc_baseaddr + XDDR_REG_CONFIG0_OFFSET);
969 num_chans = FIELD_GET(XDDR_REG_CONFIG0_NUM_CHANS_MASK, regval);
970 num_chans++;
971
972 num_csrows = FIELD_GET(XDDR_REG_CONFIG0_NUM_RANKS_MASK, regval);
973 num_csrows *= 2;
974 if (!num_csrows)
975 num_csrows = 1;
976
977 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
978 layers[0].size = num_csrows;
979 layers[0].is_virt_csrow = true;
980 layers[1].type = EDAC_MC_LAYER_CHANNEL;
981 layers[1].size = num_chans;
982 layers[1].is_virt_csrow = false;
983
984 mci = edac_mc_alloc(edac_mc_id, ARRAY_SIZE(layers), layers,
985 sizeof(struct edac_priv));
986 if (!mci) {
987 edac_printk(KERN_ERR, EDAC_MC,
988 "Failed memory allocation for mc instance\n");
989 return -ENOMEM;
990 }
991
992 priv = mci->pvt_info;
993 priv->ddrmc_baseaddr = ddrmc_baseaddr;
994 priv->ddrmc_noc_baseaddr = ddrmc_noc_baseaddr;
995 priv->ce_cnt = 0;
996 priv->ue_cnt = 0;
997 priv->mc_id = edac_mc_id;
998
999 mc_init(mci, pdev);
1000
1001 rc = edac_mc_add_mc(mci);
1002 if (rc) {
1003 edac_printk(KERN_ERR, EDAC_MC,
1004 "Failed to register with EDAC core\n");
1005 goto free_edac_mc;
1006 }
1007
1008 rc = xlnx_register_event(PM_NOTIFY_CB, VERSAL_EVENT_ERROR_PMC_ERR1,
1009 XPM_EVENT_ERROR_MASK_DDRMC_CR | XPM_EVENT_ERROR_MASK_DDRMC_NCR |
1010 XPM_EVENT_ERROR_MASK_NOC_CR | XPM_EVENT_ERROR_MASK_NOC_NCR,
1011 false, err_callback, mci);
1012 if (rc) {
1013 if (rc == -EACCES)
1014 rc = -EPROBE_DEFER;
1015
1016 goto del_mc;
1017 }
1018
1019#ifdef CONFIG_EDAC_DEBUG
1020 create_debugfs_attributes(mci);
1021 setup_address_map(priv);
1022#endif
1023 enable_intr(priv);
1024 return rc;
1025
1026del_mc:
1027 edac_mc_del_mc(&pdev->dev);
1028free_edac_mc:
1029 edac_mc_free(mci);
1030
1031 return rc;
1032}
1033
1034static int mc_remove(struct platform_device *pdev)
1035{
1036 struct mem_ctl_info *mci = platform_get_drvdata(pdev);
1037 struct edac_priv *priv = mci->pvt_info;
1038
1039 disable_intr(priv);
1040
1041#ifdef CONFIG_EDAC_DEBUG
1042 debugfs_remove_recursive(priv->debugfs);
1043#endif
1044
1045 xlnx_unregister_event(PM_NOTIFY_CB, VERSAL_EVENT_ERROR_PMC_ERR1,
1046 XPM_EVENT_ERROR_MASK_DDRMC_CR |
1047 XPM_EVENT_ERROR_MASK_NOC_CR |
1048 XPM_EVENT_ERROR_MASK_NOC_NCR |
1049 XPM_EVENT_ERROR_MASK_DDRMC_NCR, err_callback, mci);
1050 edac_mc_del_mc(&pdev->dev);
1051 edac_mc_free(mci);
1052
1053 return 0;
1054}
1055
1056static struct platform_driver xilinx_ddr_edac_mc_driver = {
1057 .driver = {
1058 .name = "xilinx-ddrmc-edac",
1059 .of_match_table = xlnx_edac_match,
1060 },
1061 .probe = mc_probe,
1062 .remove = mc_remove,
1063};
1064
1065module_platform_driver(xilinx_ddr_edac_mc_driver);
1066
1067MODULE_AUTHOR("AMD Inc");
1068MODULE_DESCRIPTION("Xilinx DDRMC ECC driver");
1069MODULE_LICENSE("GPL");