Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Driver for Intel client SoC with integrated memory controller using IBECC
4 *
5 * Copyright (C) 2020 Intel Corporation
6 *
7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific
8 * regions of the physical memory space. It's used for memory controllers
9 * that don't support the out-of-band ECC which often needs an additional
10 * storage device to each channel for storing ECC data.
11 */
12
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/pci.h>
16#include <linux/slab.h>
17#include <linux/irq_work.h>
18#include <linux/llist.h>
19#include <linux/genalloc.h>
20#include <linux/edac.h>
21#include <linux/bits.h>
22#include <linux/io.h>
23#include <asm/mach_traps.h>
24#include <asm/nmi.h>
25#include <asm/mce.h>
26
27#include "edac_mc.h"
28#include "edac_module.h"
29
30#define IGEN6_REVISION "v2.5.1"
31
32#define EDAC_MOD_STR "igen6_edac"
33#define IGEN6_NMI_NAME "igen6_ibecc"
34
35/* Debug macros */
36#define igen6_printk(level, fmt, arg...) \
37 edac_printk(level, "igen6", fmt, ##arg)
38
39#define igen6_mc_printk(mci, level, fmt, arg...) \
40 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg)
41
42#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
43
44#define NUM_IMC 2 /* Max memory controllers */
45#define NUM_CHANNELS 2 /* Max channels */
46#define NUM_DIMMS 2 /* Max DIMMs per channel */
47
48#define _4GB BIT_ULL(32)
49
50/* Size of physical memory */
51#define TOM_OFFSET 0xa0
52/* Top of low usable DRAM */
53#define TOLUD_OFFSET 0xbc
54/* Capability register C */
55#define CAPID_C_OFFSET 0xec
56#define CAPID_C_IBECC BIT(15)
57
58/* Capability register E */
59#define CAPID_E_OFFSET 0xf0
60#define CAPID_E_IBECC BIT(12)
61#define CAPID_E_IBECC_BIT18 BIT(18)
62
63/* Error Status */
64#define ERRSTS_OFFSET 0xc8
65#define ERRSTS_CE BIT_ULL(6)
66#define ERRSTS_UE BIT_ULL(7)
67
68/* Error Command */
69#define ERRCMD_OFFSET 0xca
70#define ERRCMD_CE BIT_ULL(6)
71#define ERRCMD_UE BIT_ULL(7)
72
73/* IBECC MMIO base address */
74#define IBECC_BASE (res_cfg->ibecc_base)
75#define IBECC_ACTIVATE_OFFSET IBECC_BASE
76#define IBECC_ACTIVATE_EN BIT(0)
77
78/* IBECC error log */
79#define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset)
80#define ECC_ERROR_LOG_CE BIT_ULL(62)
81#define ECC_ERROR_LOG_UE BIT_ULL(63)
82#define ECC_ERROR_LOG_ADDR_SHIFT 5
83#define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38)
84#define ECC_ERROR_LOG_ADDR45(v) GET_BITFIELD(v, 5, 45)
85#define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61)
86
87/* Host MMIO base address */
88#define MCHBAR_OFFSET 0x48
89#define MCHBAR_EN BIT_ULL(0)
90#define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16)
91#define MCHBAR_SIZE 0x10000
92
93/* Parameters for the channel decode stage */
94#define IMC_BASE (res_cfg->imc_base)
95#define MAD_INTER_CHANNEL_OFFSET IMC_BASE
96#define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2)
97#define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3)
98#define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4)
99#define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29)
100
101/* Parameters for DRAM decode stage */
102#define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4)
103#define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0)
104
105/* DIMM characteristics */
106#define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc)
107#define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29)
108#define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8)
109#define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29)
110#define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25)
111
112/* Hash for memory controller selection */
113#define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8)
114#define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3)
115
116/* Hash for channel selection */
117#define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24)
118/* Hash for enhanced channel selection */
119#define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28)
120#define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
121#define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
122#define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28)
123
124/* Parameters for memory slice decode stage */
125#define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
126#define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
127
128static struct res_config {
129 bool machine_check;
130 int num_imc;
131 u32 imc_base;
132 u32 cmf_base;
133 u32 cmf_size;
134 u32 ms_hash_offset;
135 u32 ibecc_base;
136 u32 ibecc_error_log_offset;
137 bool (*ibecc_available)(struct pci_dev *pdev);
138 /* Extract error address logged in IBECC */
139 u64 (*err_addr)(u64 ecclog);
140 /* Convert error address logged in IBECC to system physical address */
141 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc);
142 /* Convert error address logged in IBECC to integrated memory controller address */
143 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
144} *res_cfg;
145
146struct igen6_imc {
147 int mc;
148 struct mem_ctl_info *mci;
149 struct pci_dev *pdev;
150 struct device dev;
151 void __iomem *window;
152 u64 size;
153 u64 ch_s_size;
154 int ch_l_map;
155 u64 dimm_s_size[NUM_CHANNELS];
156 u64 dimm_l_size[NUM_CHANNELS];
157 int dimm_l_map[NUM_CHANNELS];
158};
159
160static struct igen6_pvt {
161 struct igen6_imc imc[NUM_IMC];
162 u64 ms_hash;
163 u64 ms_s_size;
164 int ms_l_map;
165} *igen6_pvt;
166
167/* The top of low usable DRAM */
168static u32 igen6_tolud;
169/* The size of physical memory */
170static u64 igen6_tom;
171
172struct decoded_addr {
173 int mc;
174 u64 imc_addr;
175 u64 sys_addr;
176 int channel_idx;
177 u64 channel_addr;
178 int sub_channel_idx;
179 u64 sub_channel_addr;
180};
181
182struct ecclog_node {
183 struct llist_node llnode;
184 int mc;
185 u64 ecclog;
186};
187
188/*
189 * In the NMI handler, the driver uses the lock-less memory allocator
190 * to allocate memory to store the IBECC error logs and links the logs
191 * to the lock-less list. Delay printk() and the work of error reporting
192 * to EDAC core in a worker.
193 */
194#define ECCLOG_POOL_SIZE PAGE_SIZE
195static LLIST_HEAD(ecclog_llist);
196static struct gen_pool *ecclog_pool;
197static char ecclog_buf[ECCLOG_POOL_SIZE];
198static struct irq_work ecclog_irq_work;
199static struct work_struct ecclog_work;
200
201/* Compute die IDs for Elkhart Lake with IBECC */
202#define DID_EHL_SKU5 0x4514
203#define DID_EHL_SKU6 0x4528
204#define DID_EHL_SKU7 0x452a
205#define DID_EHL_SKU8 0x4516
206#define DID_EHL_SKU9 0x452c
207#define DID_EHL_SKU10 0x452e
208#define DID_EHL_SKU11 0x4532
209#define DID_EHL_SKU12 0x4518
210#define DID_EHL_SKU13 0x451a
211#define DID_EHL_SKU14 0x4534
212#define DID_EHL_SKU15 0x4536
213
214/* Compute die IDs for ICL-NNPI with IBECC */
215#define DID_ICL_SKU8 0x4581
216#define DID_ICL_SKU10 0x4585
217#define DID_ICL_SKU11 0x4589
218#define DID_ICL_SKU12 0x458d
219
220/* Compute die IDs for Tiger Lake with IBECC */
221#define DID_TGL_SKU 0x9a14
222
223/* Compute die IDs for Alder Lake with IBECC */
224#define DID_ADL_SKU1 0x4601
225#define DID_ADL_SKU2 0x4602
226#define DID_ADL_SKU3 0x4621
227#define DID_ADL_SKU4 0x4641
228
229/* Compute die IDs for Alder Lake-N with IBECC */
230#define DID_ADL_N_SKU1 0x4614
231#define DID_ADL_N_SKU2 0x4617
232#define DID_ADL_N_SKU3 0x461b
233#define DID_ADL_N_SKU4 0x461c
234#define DID_ADL_N_SKU5 0x4673
235#define DID_ADL_N_SKU6 0x4674
236#define DID_ADL_N_SKU7 0x4675
237#define DID_ADL_N_SKU8 0x4677
238#define DID_ADL_N_SKU9 0x4678
239#define DID_ADL_N_SKU10 0x4679
240#define DID_ADL_N_SKU11 0x467c
241
242/* Compute die IDs for Raptor Lake-P with IBECC */
243#define DID_RPL_P_SKU1 0xa706
244#define DID_RPL_P_SKU2 0xa707
245#define DID_RPL_P_SKU3 0xa708
246#define DID_RPL_P_SKU4 0xa716
247#define DID_RPL_P_SKU5 0xa718
248
249/* Compute die IDs for Meteor Lake-PS with IBECC */
250#define DID_MTL_PS_SKU1 0x7d21
251#define DID_MTL_PS_SKU2 0x7d22
252#define DID_MTL_PS_SKU3 0x7d23
253#define DID_MTL_PS_SKU4 0x7d24
254
255/* Compute die IDs for Meteor Lake-P with IBECC */
256#define DID_MTL_P_SKU1 0x7d01
257#define DID_MTL_P_SKU2 0x7d02
258#define DID_MTL_P_SKU3 0x7d14
259
260static int get_mchbar(struct pci_dev *pdev, u64 *mchbar)
261{
262 union {
263 u64 v;
264 struct {
265 u32 v_lo;
266 u32 v_hi;
267 };
268 } u;
269
270 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) {
271 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n");
272 return -ENODEV;
273 }
274
275 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) {
276 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n");
277 return -ENODEV;
278 }
279
280 if (!(u.v & MCHBAR_EN)) {
281 igen6_printk(KERN_ERR, "MCHBAR is disabled\n");
282 return -ENODEV;
283 }
284
285 *mchbar = MCHBAR_BASE(u.v);
286
287 return 0;
288}
289
290static bool ehl_ibecc_available(struct pci_dev *pdev)
291{
292 u32 v;
293
294 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
295 return false;
296
297 return !!(CAPID_C_IBECC & v);
298}
299
300static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc)
301{
302 return eaddr;
303}
304
305static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc)
306{
307 if (eaddr < igen6_tolud)
308 return eaddr;
309
310 if (igen6_tom <= _4GB)
311 return eaddr + igen6_tolud - _4GB;
312
313 if (eaddr < _4GB)
314 return eaddr + igen6_tolud - igen6_tom;
315
316 return eaddr;
317}
318
319static bool icl_ibecc_available(struct pci_dev *pdev)
320{
321 u32 v;
322
323 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
324 return false;
325
326 return !(CAPID_C_IBECC & v) &&
327 (boot_cpu_data.x86_stepping >= 1);
328}
329
330static bool tgl_ibecc_available(struct pci_dev *pdev)
331{
332 u32 v;
333
334 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
335 return false;
336
337 return !(CAPID_E_IBECC & v);
338}
339
340static bool mtl_p_ibecc_available(struct pci_dev *pdev)
341{
342 u32 v;
343
344 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
345 return false;
346
347 return !(CAPID_E_IBECC_BIT18 & v);
348}
349
350static bool mtl_ps_ibecc_available(struct pci_dev *pdev)
351{
352#define MCHBAR_MEMSS_IBECCDIS 0x13c00
353 void __iomem *window;
354 u64 mchbar;
355 u32 val;
356
357 if (get_mchbar(pdev, &mchbar))
358 return false;
359
360 window = ioremap(mchbar, MCHBAR_SIZE * 2);
361 if (!window) {
362 igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar);
363 return false;
364 }
365
366 val = readl(window + MCHBAR_MEMSS_IBECCDIS);
367 iounmap(window);
368
369 /* Bit6: 1 - IBECC is disabled, 0 - IBECC isn't disabled */
370 return !GET_BITFIELD(val, 6, 6);
371}
372
373static u64 mem_addr_to_sys_addr(u64 maddr)
374{
375 if (maddr < igen6_tolud)
376 return maddr;
377
378 if (igen6_tom <= _4GB)
379 return maddr - igen6_tolud + _4GB;
380
381 if (maddr < _4GB)
382 return maddr - igen6_tolud + igen6_tom;
383
384 return maddr;
385}
386
387static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit)
388{
389 u64 hash_addr = addr & mask, hash = hash_init;
390 u64 intlv = (addr >> intlv_bit) & 1;
391 int i;
392
393 for (i = 6; i < 20; i++)
394 hash ^= (hash_addr >> i) & 1;
395
396 return hash ^ intlv;
397}
398
399static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc)
400{
401 u64 maddr, hash, mask, ms_s_size;
402 int intlv_bit;
403 u32 ms_hash;
404
405 ms_s_size = igen6_pvt->ms_s_size;
406 if (eaddr >= ms_s_size)
407 return eaddr + ms_s_size;
408
409 ms_hash = igen6_pvt->ms_hash;
410
411 mask = MEM_SLICE_HASH_MASK(ms_hash);
412 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6;
413
414 maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) |
415 GET_BITFIELD(eaddr, 0, intlv_bit - 1);
416
417 hash = mem_slice_hash(maddr, mask, mc, intlv_bit);
418
419 return maddr | (hash << intlv_bit);
420}
421
422static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc)
423{
424 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc);
425
426 return mem_addr_to_sys_addr(maddr);
427}
428
429static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc)
430{
431 return eaddr;
432}
433
434static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc)
435{
436 return mem_addr_to_sys_addr(eaddr);
437}
438
439static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc)
440{
441 u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size;
442 struct igen6_imc *imc = &igen6_pvt->imc[mc];
443 int intlv_bit;
444 u32 mc_hash;
445
446 if (eaddr >= 2 * ms_s_size)
447 return eaddr - ms_s_size;
448
449 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET);
450
451 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6;
452
453 imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit |
454 GET_BITFIELD(eaddr, 0, intlv_bit - 1);
455
456 return imc_addr;
457}
458
459static u64 rpl_p_err_addr(u64 ecclog)
460{
461 return ECC_ERROR_LOG_ADDR45(ecclog);
462}
463
464static struct res_config ehl_cfg = {
465 .num_imc = 1,
466 .imc_base = 0x5000,
467 .ibecc_base = 0xdc00,
468 .ibecc_available = ehl_ibecc_available,
469 .ibecc_error_log_offset = 0x170,
470 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
471 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
472};
473
474static struct res_config icl_cfg = {
475 .num_imc = 1,
476 .imc_base = 0x5000,
477 .ibecc_base = 0xd800,
478 .ibecc_error_log_offset = 0x170,
479 .ibecc_available = icl_ibecc_available,
480 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
481 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
482};
483
484static struct res_config tgl_cfg = {
485 .machine_check = true,
486 .num_imc = 2,
487 .imc_base = 0x5000,
488 .cmf_base = 0x11000,
489 .cmf_size = 0x800,
490 .ms_hash_offset = 0xac,
491 .ibecc_base = 0xd400,
492 .ibecc_error_log_offset = 0x170,
493 .ibecc_available = tgl_ibecc_available,
494 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr,
495 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr,
496};
497
498static struct res_config adl_cfg = {
499 .machine_check = true,
500 .num_imc = 2,
501 .imc_base = 0xd800,
502 .ibecc_base = 0xd400,
503 .ibecc_error_log_offset = 0x68,
504 .ibecc_available = tgl_ibecc_available,
505 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
506 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
507};
508
509static struct res_config adl_n_cfg = {
510 .machine_check = true,
511 .num_imc = 1,
512 .imc_base = 0xd800,
513 .ibecc_base = 0xd400,
514 .ibecc_error_log_offset = 0x68,
515 .ibecc_available = tgl_ibecc_available,
516 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
517 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
518};
519
520static struct res_config rpl_p_cfg = {
521 .machine_check = true,
522 .num_imc = 2,
523 .imc_base = 0xd800,
524 .ibecc_base = 0xd400,
525 .ibecc_error_log_offset = 0x68,
526 .ibecc_available = tgl_ibecc_available,
527 .err_addr = rpl_p_err_addr,
528 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
529 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
530};
531
532static struct res_config mtl_ps_cfg = {
533 .machine_check = true,
534 .num_imc = 2,
535 .imc_base = 0xd800,
536 .ibecc_base = 0xd400,
537 .ibecc_error_log_offset = 0x170,
538 .ibecc_available = mtl_ps_ibecc_available,
539 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
540 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
541};
542
543static struct res_config mtl_p_cfg = {
544 .machine_check = true,
545 .num_imc = 2,
546 .imc_base = 0xd800,
547 .ibecc_base = 0xd400,
548 .ibecc_error_log_offset = 0x170,
549 .ibecc_available = mtl_p_ibecc_available,
550 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
551 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
552};
553
554static const struct pci_device_id igen6_pci_tbl[] = {
555 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg },
556 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg },
557 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg },
558 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg },
559 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg },
560 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg },
561 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg },
562 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg },
563 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg },
564 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg },
565 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg },
566 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg },
567 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg },
568 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg },
569 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg },
570 { PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg },
571 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg },
572 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg },
573 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg },
574 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg },
575 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), (kernel_ulong_t)&adl_n_cfg },
576 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), (kernel_ulong_t)&adl_n_cfg },
577 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), (kernel_ulong_t)&adl_n_cfg },
578 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), (kernel_ulong_t)&adl_n_cfg },
579 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), (kernel_ulong_t)&adl_n_cfg },
580 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), (kernel_ulong_t)&adl_n_cfg },
581 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), (kernel_ulong_t)&adl_n_cfg },
582 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), (kernel_ulong_t)&adl_n_cfg },
583 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), (kernel_ulong_t)&adl_n_cfg },
584 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), (kernel_ulong_t)&adl_n_cfg },
585 { PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), (kernel_ulong_t)&adl_n_cfg },
586 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg },
587 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg },
588 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg },
589 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), (kernel_ulong_t)&rpl_p_cfg },
590 { PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), (kernel_ulong_t)&rpl_p_cfg },
591 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), (kernel_ulong_t)&mtl_ps_cfg },
592 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), (kernel_ulong_t)&mtl_ps_cfg },
593 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), (kernel_ulong_t)&mtl_ps_cfg },
594 { PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), (kernel_ulong_t)&mtl_ps_cfg },
595 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), (kernel_ulong_t)&mtl_p_cfg },
596 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), (kernel_ulong_t)&mtl_p_cfg },
597 { PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), (kernel_ulong_t)&mtl_p_cfg },
598 { },
599};
600MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
601
602static enum dev_type get_width(int dimm_l, u32 mad_dimm)
603{
604 u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) :
605 MAD_DIMM_CH_DSW(mad_dimm);
606
607 switch (w) {
608 case 0:
609 return DEV_X8;
610 case 1:
611 return DEV_X16;
612 case 2:
613 return DEV_X32;
614 default:
615 return DEV_UNKNOWN;
616 }
617}
618
619static enum mem_type get_memory_type(u32 mad_inter)
620{
621 u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter);
622
623 switch (t) {
624 case 0:
625 return MEM_DDR4;
626 case 1:
627 return MEM_DDR3;
628 case 2:
629 return MEM_LPDDR3;
630 case 3:
631 return MEM_LPDDR4;
632 case 4:
633 return MEM_WIO2;
634 default:
635 return MEM_UNKNOWN;
636 }
637}
638
639static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit)
640{
641 u64 hash_addr = addr & mask, hash = 0;
642 u64 intlv = (addr >> intlv_bit) & 1;
643 int i;
644
645 for (i = 6; i < 20; i++)
646 hash ^= (hash_addr >> i) & 1;
647
648 return (int)hash ^ intlv;
649}
650
651static u64 decode_channel_addr(u64 addr, int intlv_bit)
652{
653 u64 channel_addr;
654
655 /* Remove the interleave bit and shift upper part down to fill gap */
656 channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit;
657 channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
658
659 return channel_addr;
660}
661
662static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map,
663 int *idx, u64 *sub_addr)
664{
665 int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
666
667 if (addr > 2 * s_size) {
668 *sub_addr = addr - s_size;
669 *idx = l_map;
670 return;
671 }
672
673 if (CHANNEL_HASH_MODE(hash)) {
674 *sub_addr = decode_channel_addr(addr, intlv_bit);
675 *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit);
676 } else {
677 *sub_addr = decode_channel_addr(addr, 6);
678 *idx = GET_BITFIELD(addr, 6, 6);
679 }
680}
681
682static int igen6_decode(struct decoded_addr *res)
683{
684 struct igen6_imc *imc = &igen6_pvt->imc[res->mc];
685 u64 addr = res->imc_addr, sub_addr, s_size;
686 int idx, l_map;
687 u32 hash;
688
689 if (addr >= igen6_tom) {
690 edac_dbg(0, "Address 0x%llx out of range\n", addr);
691 return -EINVAL;
692 }
693
694 /* Decode channel */
695 hash = readl(imc->window + CHANNEL_HASH_OFFSET);
696 s_size = imc->ch_s_size;
697 l_map = imc->ch_l_map;
698 decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr);
699 res->channel_idx = idx;
700 res->channel_addr = sub_addr;
701
702 /* Decode sub-channel/DIMM */
703 hash = readl(imc->window + CHANNEL_EHASH_OFFSET);
704 s_size = imc->dimm_s_size[idx];
705 l_map = imc->dimm_l_map[idx];
706 decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr);
707 res->sub_channel_idx = idx;
708 res->sub_channel_addr = sub_addr;
709
710 return 0;
711}
712
713static void igen6_output_error(struct decoded_addr *res,
714 struct mem_ctl_info *mci, u64 ecclog)
715{
716 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ?
717 HW_EVENT_ERR_UNCORRECTED :
718 HW_EVENT_ERR_CORRECTED;
719
720 edac_mc_handle_error(type, mci, 1,
721 res->sys_addr >> PAGE_SHIFT,
722 res->sys_addr & ~PAGE_MASK,
723 ECC_ERROR_LOG_SYND(ecclog),
724 res->channel_idx, res->sub_channel_idx,
725 -1, "", "");
726}
727
728static struct gen_pool *ecclog_gen_pool_create(void)
729{
730 struct gen_pool *pool;
731
732 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1);
733 if (!pool)
734 return NULL;
735
736 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) {
737 gen_pool_destroy(pool);
738 return NULL;
739 }
740
741 return pool;
742}
743
744static int ecclog_gen_pool_add(int mc, u64 ecclog)
745{
746 struct ecclog_node *node;
747
748 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node));
749 if (!node)
750 return -ENOMEM;
751
752 node->mc = mc;
753 node->ecclog = ecclog;
754 llist_add(&node->llnode, &ecclog_llist);
755
756 return 0;
757}
758
759/*
760 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI
761 * configuration space status register ERRSTS can indicate whether a
762 * correctable error or an uncorrectable error occurred. We only use the
763 * ECC_ERROR_LOG register to check error type, but need to clear both
764 * registers to enable future error events.
765 */
766static u64 ecclog_read_and_clear(struct igen6_imc *imc)
767{
768 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET);
769
770 if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) {
771 /* Clear CE/UE bits by writing 1s */
772 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET);
773 return ecclog;
774 }
775
776 return 0;
777}
778
779static void errsts_clear(struct igen6_imc *imc)
780{
781 u16 errsts;
782
783 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) {
784 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n");
785 return;
786 }
787
788 /* Clear CE/UE bits by writing 1s */
789 if (errsts & (ERRSTS_CE | ERRSTS_UE))
790 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts);
791}
792
793static int errcmd_enable_error_reporting(bool enable)
794{
795 struct igen6_imc *imc = &igen6_pvt->imc[0];
796 u16 errcmd;
797 int rc;
798
799 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd);
800 if (rc)
801 return rc;
802
803 if (enable)
804 errcmd |= ERRCMD_CE | ERRSTS_UE;
805 else
806 errcmd &= ~(ERRCMD_CE | ERRSTS_UE);
807
808 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd);
809 if (rc)
810 return rc;
811
812 return 0;
813}
814
815static int ecclog_handler(void)
816{
817 struct igen6_imc *imc;
818 int i, n = 0;
819 u64 ecclog;
820
821 for (i = 0; i < res_cfg->num_imc; i++) {
822 imc = &igen6_pvt->imc[i];
823
824 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
825
826 ecclog = ecclog_read_and_clear(imc);
827 if (!ecclog)
828 continue;
829
830 if (!ecclog_gen_pool_add(i, ecclog))
831 irq_work_queue(&ecclog_irq_work);
832
833 n++;
834 }
835
836 return n;
837}
838
839static void ecclog_work_cb(struct work_struct *work)
840{
841 struct ecclog_node *node, *tmp;
842 struct mem_ctl_info *mci;
843 struct llist_node *head;
844 struct decoded_addr res;
845 u64 eaddr;
846
847 head = llist_del_all(&ecclog_llist);
848 if (!head)
849 return;
850
851 llist_for_each_entry_safe(node, tmp, head, llnode) {
852 memset(&res, 0, sizeof(res));
853 if (res_cfg->err_addr)
854 eaddr = res_cfg->err_addr(node->ecclog);
855 else
856 eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) <<
857 ECC_ERROR_LOG_ADDR_SHIFT;
858 res.mc = node->mc;
859 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc);
860 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc);
861
862 mci = igen6_pvt->imc[res.mc].mci;
863
864 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog);
865 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n");
866 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr);
867
868 if (!igen6_decode(&res))
869 igen6_output_error(&res, mci, node->ecclog);
870
871 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node));
872 }
873}
874
875static void ecclog_irq_work_cb(struct irq_work *irq_work)
876{
877 int i;
878
879 for (i = 0; i < res_cfg->num_imc; i++)
880 errsts_clear(&igen6_pvt->imc[i]);
881
882 if (!llist_empty(&ecclog_llist))
883 schedule_work(&ecclog_work);
884}
885
886static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs)
887{
888 unsigned char reason;
889
890 if (!ecclog_handler())
891 return NMI_DONE;
892
893 /*
894 * Both In-Band ECC correctable error and uncorrectable error are
895 * reported by SERR# NMI. The NMI generic code (see pci_serr_error())
896 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to
897 * re-enable the SERR# NMI after NMI handling. So clear this bit here
898 * to re-enable SERR# NMI for receiving future In-Band ECC errors.
899 */
900 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK;
901 reason |= NMI_REASON_CLEAR_SERR;
902 outb(reason, NMI_REASON_PORT);
903 reason &= ~NMI_REASON_CLEAR_SERR;
904 outb(reason, NMI_REASON_PORT);
905
906 return NMI_HANDLED;
907}
908
909static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val,
910 void *data)
911{
912 struct mce *mce = (struct mce *)data;
913 char *type;
914
915 if (mce->kflags & MCE_HANDLED_CEC)
916 return NOTIFY_DONE;
917
918 /*
919 * Ignore unless this is a memory related error.
920 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here,
921 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3).
922 */
923 if ((mce->status & 0xefff) >> 7 != 1)
924 return NOTIFY_DONE;
925
926 if (mce->mcgstatus & MCG_STATUS_MCIP)
927 type = "Exception";
928 else
929 type = "Event";
930
931 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n",
932 mce->extcpu, type, mce->mcgstatus,
933 mce->bank, mce->status);
934 edac_dbg(0, "TSC 0x%llx\n", mce->tsc);
935 edac_dbg(0, "ADDR 0x%llx\n", mce->addr);
936 edac_dbg(0, "MISC 0x%llx\n", mce->misc);
937 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n",
938 mce->cpuvendor, mce->cpuid, mce->time,
939 mce->socketid, mce->apicid);
940 /*
941 * We just use the Machine Check for the memory error notification.
942 * Each memory controller is associated with an IBECC instance.
943 * Directly read and clear the error information(error address and
944 * error type) on all the IBECC instances so that we know on which
945 * memory controller the memory error(s) occurred.
946 */
947 if (!ecclog_handler())
948 return NOTIFY_DONE;
949
950 mce->kflags |= MCE_HANDLED_EDAC;
951
952 return NOTIFY_DONE;
953}
954
955static struct notifier_block ecclog_mce_dec = {
956 .notifier_call = ecclog_mce_handler,
957 .priority = MCE_PRIO_EDAC,
958};
959
960static bool igen6_check_ecc(struct igen6_imc *imc)
961{
962 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET);
963
964 return !!(activate & IBECC_ACTIVATE_EN);
965}
966
967static int igen6_get_dimm_config(struct mem_ctl_info *mci)
968{
969 struct igen6_imc *imc = mci->pvt_info;
970 u32 mad_inter, mad_intra, mad_dimm;
971 int i, j, ndimms, mc = imc->mc;
972 struct dimm_info *dimm;
973 enum mem_type mtype;
974 enum dev_type dtype;
975 u64 dsize;
976 bool ecc;
977
978 edac_dbg(2, "\n");
979
980 mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
981 mtype = get_memory_type(mad_inter);
982 ecc = igen6_check_ecc(imc);
983 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter);
984 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter);
985
986 for (i = 0; i < NUM_CHANNELS; i++) {
987 mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4);
988 mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4);
989
990 imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm);
991 imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm);
992 imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra);
993 imc->size += imc->dimm_s_size[i];
994 imc->size += imc->dimm_l_size[i];
995 ndimms = 0;
996
997 for (j = 0; j < NUM_DIMMS; j++) {
998 dimm = edac_get_dimm(mci, i, j, 0);
999
1000 if (j ^ imc->dimm_l_map[i]) {
1001 dtype = get_width(0, mad_dimm);
1002 dsize = imc->dimm_s_size[i];
1003 } else {
1004 dtype = get_width(1, mad_dimm);
1005 dsize = imc->dimm_l_size[i];
1006 }
1007
1008 if (!dsize)
1009 continue;
1010
1011 dimm->grain = 64;
1012 dimm->mtype = mtype;
1013 dimm->dtype = dtype;
1014 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20);
1015 dimm->edac_mode = EDAC_SECDED;
1016 snprintf(dimm->label, sizeof(dimm->label),
1017 "MC#%d_Chan#%d_DIMM#%d", mc, i, j);
1018 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n",
1019 mc, i, j, dsize >> 20, dimm->nr_pages);
1020
1021 ndimms++;
1022 }
1023
1024 if (ndimms && !ecc) {
1025 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc);
1026 return -ENODEV;
1027 }
1028 }
1029
1030 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20);
1031
1032 return 0;
1033}
1034
1035#ifdef CONFIG_EDAC_DEBUG
1036/* Top of upper usable DRAM */
1037static u64 igen6_touud;
1038#define TOUUD_OFFSET 0xa8
1039
1040static void igen6_reg_dump(struct igen6_imc *imc)
1041{
1042 int i;
1043
1044 edac_dbg(2, "CHANNEL_HASH : 0x%x\n",
1045 readl(imc->window + CHANNEL_HASH_OFFSET));
1046 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n",
1047 readl(imc->window + CHANNEL_EHASH_OFFSET));
1048 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n",
1049 readl(imc->window + MAD_INTER_CHANNEL_OFFSET));
1050 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n",
1051 readq(imc->window + ECC_ERROR_LOG_OFFSET));
1052
1053 for (i = 0; i < NUM_CHANNELS; i++) {
1054 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i,
1055 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4));
1056 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i,
1057 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4));
1058 }
1059 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud);
1060 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud);
1061 edac_dbg(2, "TOM : 0x%llx", igen6_tom);
1062}
1063
1064static struct dentry *igen6_test;
1065
1066static int debugfs_u64_set(void *data, u64 val)
1067{
1068 u64 ecclog;
1069
1070 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) {
1071 edac_dbg(0, "Address 0x%llx out of range\n", val);
1072 return 0;
1073 }
1074
1075 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
1076
1077 val >>= ECC_ERROR_LOG_ADDR_SHIFT;
1078 ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE;
1079
1080 if (!ecclog_gen_pool_add(0, ecclog))
1081 irq_work_queue(&ecclog_irq_work);
1082
1083 return 0;
1084}
1085DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
1086
1087static void igen6_debug_setup(void)
1088{
1089 igen6_test = edac_debugfs_create_dir("igen6_test");
1090 if (!igen6_test)
1091 return;
1092
1093 if (!edac_debugfs_create_file("addr", 0200, igen6_test,
1094 NULL, &fops_u64_wo)) {
1095 debugfs_remove(igen6_test);
1096 igen6_test = NULL;
1097 }
1098}
1099
1100static void igen6_debug_teardown(void)
1101{
1102 debugfs_remove_recursive(igen6_test);
1103}
1104#else
1105static void igen6_reg_dump(struct igen6_imc *imc) {}
1106static void igen6_debug_setup(void) {}
1107static void igen6_debug_teardown(void) {}
1108#endif
1109
1110static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
1111{
1112 union {
1113 u64 v;
1114 struct {
1115 u32 v_lo;
1116 u32 v_hi;
1117 };
1118 } u;
1119
1120 edac_dbg(2, "\n");
1121
1122 if (!res_cfg->ibecc_available(pdev)) {
1123 edac_dbg(2, "No In-Band ECC IP\n");
1124 goto fail;
1125 }
1126
1127 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) {
1128 igen6_printk(KERN_ERR, "Failed to read TOLUD\n");
1129 goto fail;
1130 }
1131
1132 igen6_tolud &= GENMASK(31, 20);
1133
1134 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) {
1135 igen6_printk(KERN_ERR, "Failed to read lower TOM\n");
1136 goto fail;
1137 }
1138
1139 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) {
1140 igen6_printk(KERN_ERR, "Failed to read upper TOM\n");
1141 goto fail;
1142 }
1143
1144 igen6_tom = u.v & GENMASK_ULL(38, 20);
1145
1146 if (get_mchbar(pdev, mchbar))
1147 goto fail;
1148
1149#ifdef CONFIG_EDAC_DEBUG
1150 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo))
1151 edac_dbg(2, "Failed to read lower TOUUD\n");
1152 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi))
1153 edac_dbg(2, "Failed to read upper TOUUD\n");
1154 else
1155 igen6_touud = u.v & GENMASK_ULL(38, 20);
1156#endif
1157
1158 return 0;
1159fail:
1160 return -ENODEV;
1161}
1162
1163static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev)
1164{
1165 struct edac_mc_layer layers[2];
1166 struct mem_ctl_info *mci;
1167 struct igen6_imc *imc;
1168 void __iomem *window;
1169 int rc;
1170
1171 edac_dbg(2, "\n");
1172
1173 mchbar += mc * MCHBAR_SIZE;
1174 window = ioremap(mchbar, MCHBAR_SIZE);
1175 if (!window) {
1176 igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar);
1177 return -ENODEV;
1178 }
1179
1180 layers[0].type = EDAC_MC_LAYER_CHANNEL;
1181 layers[0].size = NUM_CHANNELS;
1182 layers[0].is_virt_csrow = false;
1183 layers[1].type = EDAC_MC_LAYER_SLOT;
1184 layers[1].size = NUM_DIMMS;
1185 layers[1].is_virt_csrow = true;
1186
1187 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0);
1188 if (!mci) {
1189 rc = -ENOMEM;
1190 goto fail;
1191 }
1192
1193 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc);
1194 if (!mci->ctl_name) {
1195 rc = -ENOMEM;
1196 goto fail2;
1197 }
1198
1199 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4;
1200 mci->edac_ctl_cap = EDAC_FLAG_SECDED;
1201 mci->edac_cap = EDAC_FLAG_SECDED;
1202 mci->mod_name = EDAC_MOD_STR;
1203 mci->dev_name = pci_name(pdev);
1204 mci->pvt_info = &igen6_pvt->imc[mc];
1205
1206 imc = mci->pvt_info;
1207 device_initialize(&imc->dev);
1208 /*
1209 * EDAC core uses mci->pdev(pointer of structure device) as
1210 * memory controller ID. The client SoCs attach one or more
1211 * memory controllers to single pci_dev (single pci_dev->dev
1212 * can be for multiple memory controllers).
1213 *
1214 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev
1215 * for the first memory controller and assign a unique imc->dev
1216 * to mci->pdev for each non-first memory controller.
1217 */
1218 mci->pdev = mc ? &imc->dev : &pdev->dev;
1219 imc->mc = mc;
1220 imc->pdev = pdev;
1221 imc->window = window;
1222
1223 igen6_reg_dump(imc);
1224
1225 rc = igen6_get_dimm_config(mci);
1226 if (rc)
1227 goto fail3;
1228
1229 rc = edac_mc_add_mc(mci);
1230 if (rc) {
1231 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc);
1232 goto fail3;
1233 }
1234
1235 imc->mci = mci;
1236 return 0;
1237fail3:
1238 kfree(mci->ctl_name);
1239fail2:
1240 edac_mc_free(mci);
1241fail:
1242 iounmap(window);
1243 return rc;
1244}
1245
1246static void igen6_unregister_mcis(void)
1247{
1248 struct mem_ctl_info *mci;
1249 struct igen6_imc *imc;
1250 int i;
1251
1252 edac_dbg(2, "\n");
1253
1254 for (i = 0; i < res_cfg->num_imc; i++) {
1255 imc = &igen6_pvt->imc[i];
1256 mci = imc->mci;
1257 if (!mci)
1258 continue;
1259
1260 edac_mc_del_mc(mci->pdev);
1261 kfree(mci->ctl_name);
1262 edac_mc_free(mci);
1263 iounmap(imc->window);
1264 }
1265}
1266
1267static int igen6_mem_slice_setup(u64 mchbar)
1268{
1269 struct igen6_imc *imc = &igen6_pvt->imc[0];
1270 u64 base = mchbar + res_cfg->cmf_base;
1271 u32 offset = res_cfg->ms_hash_offset;
1272 u32 size = res_cfg->cmf_size;
1273 u64 ms_s_size, ms_hash;
1274 void __iomem *cmf;
1275 int ms_l_map;
1276
1277 edac_dbg(2, "\n");
1278
1279 if (imc[0].size < imc[1].size) {
1280 ms_s_size = imc[0].size;
1281 ms_l_map = 1;
1282 } else {
1283 ms_s_size = imc[1].size;
1284 ms_l_map = 0;
1285 }
1286
1287 igen6_pvt->ms_s_size = ms_s_size;
1288 igen6_pvt->ms_l_map = ms_l_map;
1289
1290 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n",
1291 ms_s_size >> 20, ms_l_map);
1292
1293 if (!size)
1294 return 0;
1295
1296 cmf = ioremap(base, size);
1297 if (!cmf) {
1298 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base);
1299 return -ENODEV;
1300 }
1301
1302 ms_hash = readq(cmf + offset);
1303 igen6_pvt->ms_hash = ms_hash;
1304
1305 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash);
1306
1307 iounmap(cmf);
1308
1309 return 0;
1310}
1311
1312static int register_err_handler(void)
1313{
1314 int rc;
1315
1316 if (res_cfg->machine_check) {
1317 mce_register_decode_chain(&ecclog_mce_dec);
1318 return 0;
1319 }
1320
1321 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler,
1322 0, IGEN6_NMI_NAME);
1323 if (rc) {
1324 igen6_printk(KERN_ERR, "Failed to register NMI handler\n");
1325 return rc;
1326 }
1327
1328 return 0;
1329}
1330
1331static void unregister_err_handler(void)
1332{
1333 if (res_cfg->machine_check) {
1334 mce_unregister_decode_chain(&ecclog_mce_dec);
1335 return;
1336 }
1337
1338 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1339}
1340
1341static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1342{
1343 u64 mchbar;
1344 int i, rc;
1345
1346 edac_dbg(2, "\n");
1347
1348 igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL);
1349 if (!igen6_pvt)
1350 return -ENOMEM;
1351
1352 res_cfg = (struct res_config *)ent->driver_data;
1353
1354 rc = igen6_pci_setup(pdev, &mchbar);
1355 if (rc)
1356 goto fail;
1357
1358 for (i = 0; i < res_cfg->num_imc; i++) {
1359 rc = igen6_register_mci(i, mchbar, pdev);
1360 if (rc)
1361 goto fail2;
1362 }
1363
1364 if (res_cfg->num_imc > 1) {
1365 rc = igen6_mem_slice_setup(mchbar);
1366 if (rc)
1367 goto fail2;
1368 }
1369
1370 ecclog_pool = ecclog_gen_pool_create();
1371 if (!ecclog_pool) {
1372 rc = -ENOMEM;
1373 goto fail2;
1374 }
1375
1376 INIT_WORK(&ecclog_work, ecclog_work_cb);
1377 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb);
1378
1379 rc = register_err_handler();
1380 if (rc)
1381 goto fail3;
1382
1383 /* Enable error reporting */
1384 rc = errcmd_enable_error_reporting(true);
1385 if (rc) {
1386 igen6_printk(KERN_ERR, "Failed to enable error reporting\n");
1387 goto fail4;
1388 }
1389
1390 /* Check if any pending errors before/during the registration of the error handler */
1391 ecclog_handler();
1392
1393 igen6_debug_setup();
1394 return 0;
1395fail4:
1396 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1397fail3:
1398 gen_pool_destroy(ecclog_pool);
1399fail2:
1400 igen6_unregister_mcis();
1401fail:
1402 kfree(igen6_pvt);
1403 return rc;
1404}
1405
1406static void igen6_remove(struct pci_dev *pdev)
1407{
1408 edac_dbg(2, "\n");
1409
1410 igen6_debug_teardown();
1411 errcmd_enable_error_reporting(false);
1412 unregister_err_handler();
1413 irq_work_sync(&ecclog_irq_work);
1414 flush_work(&ecclog_work);
1415 gen_pool_destroy(ecclog_pool);
1416 igen6_unregister_mcis();
1417 kfree(igen6_pvt);
1418}
1419
1420static struct pci_driver igen6_driver = {
1421 .name = EDAC_MOD_STR,
1422 .probe = igen6_probe,
1423 .remove = igen6_remove,
1424 .id_table = igen6_pci_tbl,
1425};
1426
1427static int __init igen6_init(void)
1428{
1429 const char *owner;
1430 int rc;
1431
1432 edac_dbg(2, "\n");
1433
1434 if (ghes_get_devices())
1435 return -EBUSY;
1436
1437 owner = edac_get_owner();
1438 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
1439 return -EBUSY;
1440
1441 edac_op_state = EDAC_OPSTATE_NMI;
1442
1443 rc = pci_register_driver(&igen6_driver);
1444 if (rc)
1445 return rc;
1446
1447 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION);
1448
1449 return 0;
1450}
1451
1452static void __exit igen6_exit(void)
1453{
1454 edac_dbg(2, "\n");
1455
1456 pci_unregister_driver(&igen6_driver);
1457}
1458
1459module_init(igen6_init);
1460module_exit(igen6_exit);
1461
1462MODULE_LICENSE("GPL v2");
1463MODULE_AUTHOR("Qiuxu Zhuo");
1464MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC");
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Driver for Intel client SoC with integrated memory controller using IBECC
4 *
5 * Copyright (C) 2020 Intel Corporation
6 *
7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific
8 * regions of the physical memory space. It's used for memory controllers
9 * that don't support the out-of-band ECC which often needs an additional
10 * storage device to each channel for storing ECC data.
11 */
12
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/pci.h>
16#include <linux/slab.h>
17#include <linux/irq_work.h>
18#include <linux/llist.h>
19#include <linux/genalloc.h>
20#include <linux/edac.h>
21#include <linux/bits.h>
22#include <linux/io.h>
23#include <asm/mach_traps.h>
24#include <asm/nmi.h>
25#include <asm/mce.h>
26
27#include "edac_mc.h"
28#include "edac_module.h"
29
30#define IGEN6_REVISION "v2.5"
31
32#define EDAC_MOD_STR "igen6_edac"
33#define IGEN6_NMI_NAME "igen6_ibecc"
34
35/* Debug macros */
36#define igen6_printk(level, fmt, arg...) \
37 edac_printk(level, "igen6", fmt, ##arg)
38
39#define igen6_mc_printk(mci, level, fmt, arg...) \
40 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg)
41
42#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
43
44#define NUM_IMC 2 /* Max memory controllers */
45#define NUM_CHANNELS 2 /* Max channels */
46#define NUM_DIMMS 2 /* Max DIMMs per channel */
47
48#define _4GB BIT_ULL(32)
49
50/* Size of physical memory */
51#define TOM_OFFSET 0xa0
52/* Top of low usable DRAM */
53#define TOLUD_OFFSET 0xbc
54/* Capability register C */
55#define CAPID_C_OFFSET 0xec
56#define CAPID_C_IBECC BIT(15)
57
58/* Capability register E */
59#define CAPID_E_OFFSET 0xf0
60#define CAPID_E_IBECC BIT(12)
61
62/* Error Status */
63#define ERRSTS_OFFSET 0xc8
64#define ERRSTS_CE BIT_ULL(6)
65#define ERRSTS_UE BIT_ULL(7)
66
67/* Error Command */
68#define ERRCMD_OFFSET 0xca
69#define ERRCMD_CE BIT_ULL(6)
70#define ERRCMD_UE BIT_ULL(7)
71
72/* IBECC MMIO base address */
73#define IBECC_BASE (res_cfg->ibecc_base)
74#define IBECC_ACTIVATE_OFFSET IBECC_BASE
75#define IBECC_ACTIVATE_EN BIT(0)
76
77/* IBECC error log */
78#define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset)
79#define ECC_ERROR_LOG_CE BIT_ULL(62)
80#define ECC_ERROR_LOG_UE BIT_ULL(63)
81#define ECC_ERROR_LOG_ADDR_SHIFT 5
82#define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38)
83#define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61)
84
85/* Host MMIO base address */
86#define MCHBAR_OFFSET 0x48
87#define MCHBAR_EN BIT_ULL(0)
88#define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16)
89#define MCHBAR_SIZE 0x10000
90
91/* Parameters for the channel decode stage */
92#define IMC_BASE (res_cfg->imc_base)
93#define MAD_INTER_CHANNEL_OFFSET IMC_BASE
94#define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2)
95#define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3)
96#define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4)
97#define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29)
98
99/* Parameters for DRAM decode stage */
100#define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4)
101#define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0)
102
103/* DIMM characteristics */
104#define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc)
105#define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29)
106#define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8)
107#define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29)
108#define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25)
109
110/* Hash for memory controller selection */
111#define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8)
112#define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3)
113
114/* Hash for channel selection */
115#define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24)
116/* Hash for enhanced channel selection */
117#define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28)
118#define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
119#define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
120#define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28)
121
122/* Parameters for memory slice decode stage */
123#define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6)
124#define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26)
125
126static struct res_config {
127 bool machine_check;
128 int num_imc;
129 u32 imc_base;
130 u32 cmf_base;
131 u32 cmf_size;
132 u32 ms_hash_offset;
133 u32 ibecc_base;
134 u32 ibecc_error_log_offset;
135 bool (*ibecc_available)(struct pci_dev *pdev);
136 /* Convert error address logged in IBECC to system physical address */
137 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc);
138 /* Convert error address logged in IBECC to integrated memory controller address */
139 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
140} *res_cfg;
141
142struct igen6_imc {
143 int mc;
144 struct mem_ctl_info *mci;
145 struct pci_dev *pdev;
146 struct device dev;
147 void __iomem *window;
148 u64 size;
149 u64 ch_s_size;
150 int ch_l_map;
151 u64 dimm_s_size[NUM_CHANNELS];
152 u64 dimm_l_size[NUM_CHANNELS];
153 int dimm_l_map[NUM_CHANNELS];
154};
155
156static struct igen6_pvt {
157 struct igen6_imc imc[NUM_IMC];
158 u64 ms_hash;
159 u64 ms_s_size;
160 int ms_l_map;
161} *igen6_pvt;
162
163/* The top of low usable DRAM */
164static u32 igen6_tolud;
165/* The size of physical memory */
166static u64 igen6_tom;
167
168struct decoded_addr {
169 int mc;
170 u64 imc_addr;
171 u64 sys_addr;
172 int channel_idx;
173 u64 channel_addr;
174 int sub_channel_idx;
175 u64 sub_channel_addr;
176};
177
178struct ecclog_node {
179 struct llist_node llnode;
180 int mc;
181 u64 ecclog;
182};
183
184/*
185 * In the NMI handler, the driver uses the lock-less memory allocator
186 * to allocate memory to store the IBECC error logs and links the logs
187 * to the lock-less list. Delay printk() and the work of error reporting
188 * to EDAC core in a worker.
189 */
190#define ECCLOG_POOL_SIZE PAGE_SIZE
191static LLIST_HEAD(ecclog_llist);
192static struct gen_pool *ecclog_pool;
193static char ecclog_buf[ECCLOG_POOL_SIZE];
194static struct irq_work ecclog_irq_work;
195static struct work_struct ecclog_work;
196
197/* Compute die IDs for Elkhart Lake with IBECC */
198#define DID_EHL_SKU5 0x4514
199#define DID_EHL_SKU6 0x4528
200#define DID_EHL_SKU7 0x452a
201#define DID_EHL_SKU8 0x4516
202#define DID_EHL_SKU9 0x452c
203#define DID_EHL_SKU10 0x452e
204#define DID_EHL_SKU11 0x4532
205#define DID_EHL_SKU12 0x4518
206#define DID_EHL_SKU13 0x451a
207#define DID_EHL_SKU14 0x4534
208#define DID_EHL_SKU15 0x4536
209
210/* Compute die IDs for ICL-NNPI with IBECC */
211#define DID_ICL_SKU8 0x4581
212#define DID_ICL_SKU10 0x4585
213#define DID_ICL_SKU11 0x4589
214#define DID_ICL_SKU12 0x458d
215
216/* Compute die IDs for Tiger Lake with IBECC */
217#define DID_TGL_SKU 0x9a14
218
219/* Compute die IDs for Alder Lake with IBECC */
220#define DID_ADL_SKU1 0x4601
221#define DID_ADL_SKU2 0x4602
222#define DID_ADL_SKU3 0x4621
223#define DID_ADL_SKU4 0x4641
224
225static bool ehl_ibecc_available(struct pci_dev *pdev)
226{
227 u32 v;
228
229 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
230 return false;
231
232 return !!(CAPID_C_IBECC & v);
233}
234
235static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc)
236{
237 return eaddr;
238}
239
240static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc)
241{
242 if (eaddr < igen6_tolud)
243 return eaddr;
244
245 if (igen6_tom <= _4GB)
246 return eaddr + igen6_tolud - _4GB;
247
248 if (eaddr < _4GB)
249 return eaddr + igen6_tolud - igen6_tom;
250
251 return eaddr;
252}
253
254static bool icl_ibecc_available(struct pci_dev *pdev)
255{
256 u32 v;
257
258 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
259 return false;
260
261 return !(CAPID_C_IBECC & v) &&
262 (boot_cpu_data.x86_stepping >= 1);
263}
264
265static bool tgl_ibecc_available(struct pci_dev *pdev)
266{
267 u32 v;
268
269 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
270 return false;
271
272 return !(CAPID_E_IBECC & v);
273}
274
275static u64 mem_addr_to_sys_addr(u64 maddr)
276{
277 if (maddr < igen6_tolud)
278 return maddr;
279
280 if (igen6_tom <= _4GB)
281 return maddr - igen6_tolud + _4GB;
282
283 if (maddr < _4GB)
284 return maddr - igen6_tolud + igen6_tom;
285
286 return maddr;
287}
288
289static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit)
290{
291 u64 hash_addr = addr & mask, hash = hash_init;
292 u64 intlv = (addr >> intlv_bit) & 1;
293 int i;
294
295 for (i = 6; i < 20; i++)
296 hash ^= (hash_addr >> i) & 1;
297
298 return hash ^ intlv;
299}
300
301static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc)
302{
303 u64 maddr, hash, mask, ms_s_size;
304 int intlv_bit;
305 u32 ms_hash;
306
307 ms_s_size = igen6_pvt->ms_s_size;
308 if (eaddr >= ms_s_size)
309 return eaddr + ms_s_size;
310
311 ms_hash = igen6_pvt->ms_hash;
312
313 mask = MEM_SLICE_HASH_MASK(ms_hash);
314 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6;
315
316 maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) |
317 GET_BITFIELD(eaddr, 0, intlv_bit - 1);
318
319 hash = mem_slice_hash(maddr, mask, mc, intlv_bit);
320
321 return maddr | (hash << intlv_bit);
322}
323
324static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc)
325{
326 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc);
327
328 return mem_addr_to_sys_addr(maddr);
329}
330
331static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc)
332{
333 return eaddr;
334}
335
336static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc)
337{
338 return mem_addr_to_sys_addr(eaddr);
339}
340
341static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc)
342{
343 u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size;
344 struct igen6_imc *imc = &igen6_pvt->imc[mc];
345 int intlv_bit;
346 u32 mc_hash;
347
348 if (eaddr >= 2 * ms_s_size)
349 return eaddr - ms_s_size;
350
351 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET);
352
353 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6;
354
355 imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit |
356 GET_BITFIELD(eaddr, 0, intlv_bit - 1);
357
358 return imc_addr;
359}
360
361static struct res_config ehl_cfg = {
362 .num_imc = 1,
363 .imc_base = 0x5000,
364 .ibecc_base = 0xdc00,
365 .ibecc_available = ehl_ibecc_available,
366 .ibecc_error_log_offset = 0x170,
367 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
368 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
369};
370
371static struct res_config icl_cfg = {
372 .num_imc = 1,
373 .imc_base = 0x5000,
374 .ibecc_base = 0xd800,
375 .ibecc_error_log_offset = 0x170,
376 .ibecc_available = icl_ibecc_available,
377 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr,
378 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr,
379};
380
381static struct res_config tgl_cfg = {
382 .machine_check = true,
383 .num_imc = 2,
384 .imc_base = 0x5000,
385 .cmf_base = 0x11000,
386 .cmf_size = 0x800,
387 .ms_hash_offset = 0xac,
388 .ibecc_base = 0xd400,
389 .ibecc_error_log_offset = 0x170,
390 .ibecc_available = tgl_ibecc_available,
391 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr,
392 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr,
393};
394
395static struct res_config adl_cfg = {
396 .machine_check = true,
397 .num_imc = 2,
398 .imc_base = 0xd800,
399 .ibecc_base = 0xd400,
400 .ibecc_error_log_offset = 0x68,
401 .ibecc_available = tgl_ibecc_available,
402 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr,
403 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr,
404};
405
406static const struct pci_device_id igen6_pci_tbl[] = {
407 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg },
408 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg },
409 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg },
410 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg },
411 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg },
412 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg },
413 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg },
414 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg },
415 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg },
416 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg },
417 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg },
418 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg },
419 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg },
420 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg },
421 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg },
422 { PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg },
423 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg },
424 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg },
425 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg },
426 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg },
427 { },
428};
429MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
430
431static enum dev_type get_width(int dimm_l, u32 mad_dimm)
432{
433 u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) :
434 MAD_DIMM_CH_DSW(mad_dimm);
435
436 switch (w) {
437 case 0:
438 return DEV_X8;
439 case 1:
440 return DEV_X16;
441 case 2:
442 return DEV_X32;
443 default:
444 return DEV_UNKNOWN;
445 }
446}
447
448static enum mem_type get_memory_type(u32 mad_inter)
449{
450 u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter);
451
452 switch (t) {
453 case 0:
454 return MEM_DDR4;
455 case 1:
456 return MEM_DDR3;
457 case 2:
458 return MEM_LPDDR3;
459 case 3:
460 return MEM_LPDDR4;
461 case 4:
462 return MEM_WIO2;
463 default:
464 return MEM_UNKNOWN;
465 }
466}
467
468static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit)
469{
470 u64 hash_addr = addr & mask, hash = 0;
471 u64 intlv = (addr >> intlv_bit) & 1;
472 int i;
473
474 for (i = 6; i < 20; i++)
475 hash ^= (hash_addr >> i) & 1;
476
477 return (int)hash ^ intlv;
478}
479
480static u64 decode_channel_addr(u64 addr, int intlv_bit)
481{
482 u64 channel_addr;
483
484 /* Remove the interleave bit and shift upper part down to fill gap */
485 channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit;
486 channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
487
488 return channel_addr;
489}
490
491static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map,
492 int *idx, u64 *sub_addr)
493{
494 int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
495
496 if (addr > 2 * s_size) {
497 *sub_addr = addr - s_size;
498 *idx = l_map;
499 return;
500 }
501
502 if (CHANNEL_HASH_MODE(hash)) {
503 *sub_addr = decode_channel_addr(addr, intlv_bit);
504 *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit);
505 } else {
506 *sub_addr = decode_channel_addr(addr, 6);
507 *idx = GET_BITFIELD(addr, 6, 6);
508 }
509}
510
511static int igen6_decode(struct decoded_addr *res)
512{
513 struct igen6_imc *imc = &igen6_pvt->imc[res->mc];
514 u64 addr = res->imc_addr, sub_addr, s_size;
515 int idx, l_map;
516 u32 hash;
517
518 if (addr >= igen6_tom) {
519 edac_dbg(0, "Address 0x%llx out of range\n", addr);
520 return -EINVAL;
521 }
522
523 /* Decode channel */
524 hash = readl(imc->window + CHANNEL_HASH_OFFSET);
525 s_size = imc->ch_s_size;
526 l_map = imc->ch_l_map;
527 decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr);
528 res->channel_idx = idx;
529 res->channel_addr = sub_addr;
530
531 /* Decode sub-channel/DIMM */
532 hash = readl(imc->window + CHANNEL_EHASH_OFFSET);
533 s_size = imc->dimm_s_size[idx];
534 l_map = imc->dimm_l_map[idx];
535 decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr);
536 res->sub_channel_idx = idx;
537 res->sub_channel_addr = sub_addr;
538
539 return 0;
540}
541
542static void igen6_output_error(struct decoded_addr *res,
543 struct mem_ctl_info *mci, u64 ecclog)
544{
545 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ?
546 HW_EVENT_ERR_UNCORRECTED :
547 HW_EVENT_ERR_CORRECTED;
548
549 edac_mc_handle_error(type, mci, 1,
550 res->sys_addr >> PAGE_SHIFT,
551 res->sys_addr & ~PAGE_MASK,
552 ECC_ERROR_LOG_SYND(ecclog),
553 res->channel_idx, res->sub_channel_idx,
554 -1, "", "");
555}
556
557static struct gen_pool *ecclog_gen_pool_create(void)
558{
559 struct gen_pool *pool;
560
561 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1);
562 if (!pool)
563 return NULL;
564
565 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) {
566 gen_pool_destroy(pool);
567 return NULL;
568 }
569
570 return pool;
571}
572
573static int ecclog_gen_pool_add(int mc, u64 ecclog)
574{
575 struct ecclog_node *node;
576
577 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node));
578 if (!node)
579 return -ENOMEM;
580
581 node->mc = mc;
582 node->ecclog = ecclog;
583 llist_add(&node->llnode, &ecclog_llist);
584
585 return 0;
586}
587
588/*
589 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI
590 * configuration space status register ERRSTS can indicate whether a
591 * correctable error or an uncorrectable error occurred. We only use the
592 * ECC_ERROR_LOG register to check error type, but need to clear both
593 * registers to enable future error events.
594 */
595static u64 ecclog_read_and_clear(struct igen6_imc *imc)
596{
597 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET);
598
599 if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) {
600 /* Clear CE/UE bits by writing 1s */
601 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET);
602 return ecclog;
603 }
604
605 return 0;
606}
607
608static void errsts_clear(struct igen6_imc *imc)
609{
610 u16 errsts;
611
612 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) {
613 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n");
614 return;
615 }
616
617 /* Clear CE/UE bits by writing 1s */
618 if (errsts & (ERRSTS_CE | ERRSTS_UE))
619 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts);
620}
621
622static int errcmd_enable_error_reporting(bool enable)
623{
624 struct igen6_imc *imc = &igen6_pvt->imc[0];
625 u16 errcmd;
626 int rc;
627
628 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd);
629 if (rc)
630 return rc;
631
632 if (enable)
633 errcmd |= ERRCMD_CE | ERRSTS_UE;
634 else
635 errcmd &= ~(ERRCMD_CE | ERRSTS_UE);
636
637 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd);
638 if (rc)
639 return rc;
640
641 return 0;
642}
643
644static int ecclog_handler(void)
645{
646 struct igen6_imc *imc;
647 int i, n = 0;
648 u64 ecclog;
649
650 for (i = 0; i < res_cfg->num_imc; i++) {
651 imc = &igen6_pvt->imc[i];
652
653 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
654
655 ecclog = ecclog_read_and_clear(imc);
656 if (!ecclog)
657 continue;
658
659 if (!ecclog_gen_pool_add(i, ecclog))
660 irq_work_queue(&ecclog_irq_work);
661
662 n++;
663 }
664
665 return n;
666}
667
668static void ecclog_work_cb(struct work_struct *work)
669{
670 struct ecclog_node *node, *tmp;
671 struct mem_ctl_info *mci;
672 struct llist_node *head;
673 struct decoded_addr res;
674 u64 eaddr;
675
676 head = llist_del_all(&ecclog_llist);
677 if (!head)
678 return;
679
680 llist_for_each_entry_safe(node, tmp, head, llnode) {
681 memset(&res, 0, sizeof(res));
682 eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) <<
683 ECC_ERROR_LOG_ADDR_SHIFT;
684 res.mc = node->mc;
685 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc);
686 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc);
687
688 mci = igen6_pvt->imc[res.mc].mci;
689
690 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog);
691 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n");
692 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr);
693
694 if (!igen6_decode(&res))
695 igen6_output_error(&res, mci, node->ecclog);
696
697 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node));
698 }
699}
700
701static void ecclog_irq_work_cb(struct irq_work *irq_work)
702{
703 int i;
704
705 for (i = 0; i < res_cfg->num_imc; i++)
706 errsts_clear(&igen6_pvt->imc[i]);
707
708 if (!llist_empty(&ecclog_llist))
709 schedule_work(&ecclog_work);
710}
711
712static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs)
713{
714 unsigned char reason;
715
716 if (!ecclog_handler())
717 return NMI_DONE;
718
719 /*
720 * Both In-Band ECC correctable error and uncorrectable error are
721 * reported by SERR# NMI. The NMI generic code (see pci_serr_error())
722 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to
723 * re-enable the SERR# NMI after NMI handling. So clear this bit here
724 * to re-enable SERR# NMI for receiving future In-Band ECC errors.
725 */
726 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK;
727 reason |= NMI_REASON_CLEAR_SERR;
728 outb(reason, NMI_REASON_PORT);
729 reason &= ~NMI_REASON_CLEAR_SERR;
730 outb(reason, NMI_REASON_PORT);
731
732 return NMI_HANDLED;
733}
734
735static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val,
736 void *data)
737{
738 struct mce *mce = (struct mce *)data;
739 char *type;
740
741 if (mce->kflags & MCE_HANDLED_CEC)
742 return NOTIFY_DONE;
743
744 /*
745 * Ignore unless this is a memory related error.
746 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here,
747 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3).
748 */
749 if ((mce->status & 0xefff) >> 7 != 1)
750 return NOTIFY_DONE;
751
752 if (mce->mcgstatus & MCG_STATUS_MCIP)
753 type = "Exception";
754 else
755 type = "Event";
756
757 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n",
758 mce->extcpu, type, mce->mcgstatus,
759 mce->bank, mce->status);
760 edac_dbg(0, "TSC 0x%llx\n", mce->tsc);
761 edac_dbg(0, "ADDR 0x%llx\n", mce->addr);
762 edac_dbg(0, "MISC 0x%llx\n", mce->misc);
763 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n",
764 mce->cpuvendor, mce->cpuid, mce->time,
765 mce->socketid, mce->apicid);
766 /*
767 * We just use the Machine Check for the memory error notification.
768 * Each memory controller is associated with an IBECC instance.
769 * Directly read and clear the error information(error address and
770 * error type) on all the IBECC instances so that we know on which
771 * memory controller the memory error(s) occurred.
772 */
773 if (!ecclog_handler())
774 return NOTIFY_DONE;
775
776 mce->kflags |= MCE_HANDLED_EDAC;
777
778 return NOTIFY_DONE;
779}
780
781static struct notifier_block ecclog_mce_dec = {
782 .notifier_call = ecclog_mce_handler,
783 .priority = MCE_PRIO_EDAC,
784};
785
786static bool igen6_check_ecc(struct igen6_imc *imc)
787{
788 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET);
789
790 return !!(activate & IBECC_ACTIVATE_EN);
791}
792
793static int igen6_get_dimm_config(struct mem_ctl_info *mci)
794{
795 struct igen6_imc *imc = mci->pvt_info;
796 u32 mad_inter, mad_intra, mad_dimm;
797 int i, j, ndimms, mc = imc->mc;
798 struct dimm_info *dimm;
799 enum mem_type mtype;
800 enum dev_type dtype;
801 u64 dsize;
802 bool ecc;
803
804 edac_dbg(2, "\n");
805
806 mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
807 mtype = get_memory_type(mad_inter);
808 ecc = igen6_check_ecc(imc);
809 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter);
810 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter);
811
812 for (i = 0; i < NUM_CHANNELS; i++) {
813 mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4);
814 mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4);
815
816 imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm);
817 imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm);
818 imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra);
819 imc->size += imc->dimm_s_size[i];
820 imc->size += imc->dimm_l_size[i];
821 ndimms = 0;
822
823 for (j = 0; j < NUM_DIMMS; j++) {
824 dimm = edac_get_dimm(mci, i, j, 0);
825
826 if (j ^ imc->dimm_l_map[i]) {
827 dtype = get_width(0, mad_dimm);
828 dsize = imc->dimm_s_size[i];
829 } else {
830 dtype = get_width(1, mad_dimm);
831 dsize = imc->dimm_l_size[i];
832 }
833
834 if (!dsize)
835 continue;
836
837 dimm->grain = 64;
838 dimm->mtype = mtype;
839 dimm->dtype = dtype;
840 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20);
841 dimm->edac_mode = EDAC_SECDED;
842 snprintf(dimm->label, sizeof(dimm->label),
843 "MC#%d_Chan#%d_DIMM#%d", mc, i, j);
844 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n",
845 mc, i, j, dsize >> 20, dimm->nr_pages);
846
847 ndimms++;
848 }
849
850 if (ndimms && !ecc) {
851 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc);
852 return -ENODEV;
853 }
854 }
855
856 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20);
857
858 return 0;
859}
860
861#ifdef CONFIG_EDAC_DEBUG
862/* Top of upper usable DRAM */
863static u64 igen6_touud;
864#define TOUUD_OFFSET 0xa8
865
866static void igen6_reg_dump(struct igen6_imc *imc)
867{
868 int i;
869
870 edac_dbg(2, "CHANNEL_HASH : 0x%x\n",
871 readl(imc->window + CHANNEL_HASH_OFFSET));
872 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n",
873 readl(imc->window + CHANNEL_EHASH_OFFSET));
874 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n",
875 readl(imc->window + MAD_INTER_CHANNEL_OFFSET));
876 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n",
877 readq(imc->window + ECC_ERROR_LOG_OFFSET));
878
879 for (i = 0; i < NUM_CHANNELS; i++) {
880 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i,
881 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4));
882 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i,
883 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4));
884 }
885 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud);
886 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud);
887 edac_dbg(2, "TOM : 0x%llx", igen6_tom);
888}
889
890static struct dentry *igen6_test;
891
892static int debugfs_u64_set(void *data, u64 val)
893{
894 u64 ecclog;
895
896 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) {
897 edac_dbg(0, "Address 0x%llx out of range\n", val);
898 return 0;
899 }
900
901 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
902
903 val >>= ECC_ERROR_LOG_ADDR_SHIFT;
904 ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE;
905
906 if (!ecclog_gen_pool_add(0, ecclog))
907 irq_work_queue(&ecclog_irq_work);
908
909 return 0;
910}
911DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
912
913static void igen6_debug_setup(void)
914{
915 igen6_test = edac_debugfs_create_dir("igen6_test");
916 if (!igen6_test)
917 return;
918
919 if (!edac_debugfs_create_file("addr", 0200, igen6_test,
920 NULL, &fops_u64_wo)) {
921 debugfs_remove(igen6_test);
922 igen6_test = NULL;
923 }
924}
925
926static void igen6_debug_teardown(void)
927{
928 debugfs_remove_recursive(igen6_test);
929}
930#else
931static void igen6_reg_dump(struct igen6_imc *imc) {}
932static void igen6_debug_setup(void) {}
933static void igen6_debug_teardown(void) {}
934#endif
935
936static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
937{
938 union {
939 u64 v;
940 struct {
941 u32 v_lo;
942 u32 v_hi;
943 };
944 } u;
945
946 edac_dbg(2, "\n");
947
948 if (!res_cfg->ibecc_available(pdev)) {
949 edac_dbg(2, "No In-Band ECC IP\n");
950 goto fail;
951 }
952
953 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) {
954 igen6_printk(KERN_ERR, "Failed to read TOLUD\n");
955 goto fail;
956 }
957
958 igen6_tolud &= GENMASK(31, 20);
959
960 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) {
961 igen6_printk(KERN_ERR, "Failed to read lower TOM\n");
962 goto fail;
963 }
964
965 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) {
966 igen6_printk(KERN_ERR, "Failed to read upper TOM\n");
967 goto fail;
968 }
969
970 igen6_tom = u.v & GENMASK_ULL(38, 20);
971
972 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) {
973 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n");
974 goto fail;
975 }
976
977 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) {
978 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n");
979 goto fail;
980 }
981
982 if (!(u.v & MCHBAR_EN)) {
983 igen6_printk(KERN_ERR, "MCHBAR is disabled\n");
984 goto fail;
985 }
986
987 *mchbar = MCHBAR_BASE(u.v);
988
989#ifdef CONFIG_EDAC_DEBUG
990 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo))
991 edac_dbg(2, "Failed to read lower TOUUD\n");
992 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi))
993 edac_dbg(2, "Failed to read upper TOUUD\n");
994 else
995 igen6_touud = u.v & GENMASK_ULL(38, 20);
996#endif
997
998 return 0;
999fail:
1000 return -ENODEV;
1001}
1002
1003static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev)
1004{
1005 struct edac_mc_layer layers[2];
1006 struct mem_ctl_info *mci;
1007 struct igen6_imc *imc;
1008 void __iomem *window;
1009 int rc;
1010
1011 edac_dbg(2, "\n");
1012
1013 mchbar += mc * MCHBAR_SIZE;
1014 window = ioremap(mchbar, MCHBAR_SIZE);
1015 if (!window) {
1016 igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar);
1017 return -ENODEV;
1018 }
1019
1020 layers[0].type = EDAC_MC_LAYER_CHANNEL;
1021 layers[0].size = NUM_CHANNELS;
1022 layers[0].is_virt_csrow = false;
1023 layers[1].type = EDAC_MC_LAYER_SLOT;
1024 layers[1].size = NUM_DIMMS;
1025 layers[1].is_virt_csrow = true;
1026
1027 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0);
1028 if (!mci) {
1029 rc = -ENOMEM;
1030 goto fail;
1031 }
1032
1033 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc);
1034 if (!mci->ctl_name) {
1035 rc = -ENOMEM;
1036 goto fail2;
1037 }
1038
1039 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4;
1040 mci->edac_ctl_cap = EDAC_FLAG_SECDED;
1041 mci->edac_cap = EDAC_FLAG_SECDED;
1042 mci->mod_name = EDAC_MOD_STR;
1043 mci->dev_name = pci_name(pdev);
1044 mci->pvt_info = &igen6_pvt->imc[mc];
1045
1046 imc = mci->pvt_info;
1047 device_initialize(&imc->dev);
1048 /*
1049 * EDAC core uses mci->pdev(pointer of structure device) as
1050 * memory controller ID. The client SoCs attach one or more
1051 * memory controllers to single pci_dev (single pci_dev->dev
1052 * can be for multiple memory controllers).
1053 *
1054 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev
1055 * for the first memory controller and assign a unique imc->dev
1056 * to mci->pdev for each non-first memory controller.
1057 */
1058 mci->pdev = mc ? &imc->dev : &pdev->dev;
1059 imc->mc = mc;
1060 imc->pdev = pdev;
1061 imc->window = window;
1062
1063 igen6_reg_dump(imc);
1064
1065 rc = igen6_get_dimm_config(mci);
1066 if (rc)
1067 goto fail3;
1068
1069 rc = edac_mc_add_mc(mci);
1070 if (rc) {
1071 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc);
1072 goto fail3;
1073 }
1074
1075 imc->mci = mci;
1076 return 0;
1077fail3:
1078 kfree(mci->ctl_name);
1079fail2:
1080 edac_mc_free(mci);
1081fail:
1082 iounmap(window);
1083 return rc;
1084}
1085
1086static void igen6_unregister_mcis(void)
1087{
1088 struct mem_ctl_info *mci;
1089 struct igen6_imc *imc;
1090 int i;
1091
1092 edac_dbg(2, "\n");
1093
1094 for (i = 0; i < res_cfg->num_imc; i++) {
1095 imc = &igen6_pvt->imc[i];
1096 mci = imc->mci;
1097 if (!mci)
1098 continue;
1099
1100 edac_mc_del_mc(mci->pdev);
1101 kfree(mci->ctl_name);
1102 edac_mc_free(mci);
1103 iounmap(imc->window);
1104 }
1105}
1106
1107static int igen6_mem_slice_setup(u64 mchbar)
1108{
1109 struct igen6_imc *imc = &igen6_pvt->imc[0];
1110 u64 base = mchbar + res_cfg->cmf_base;
1111 u32 offset = res_cfg->ms_hash_offset;
1112 u32 size = res_cfg->cmf_size;
1113 u64 ms_s_size, ms_hash;
1114 void __iomem *cmf;
1115 int ms_l_map;
1116
1117 edac_dbg(2, "\n");
1118
1119 if (imc[0].size < imc[1].size) {
1120 ms_s_size = imc[0].size;
1121 ms_l_map = 1;
1122 } else {
1123 ms_s_size = imc[1].size;
1124 ms_l_map = 0;
1125 }
1126
1127 igen6_pvt->ms_s_size = ms_s_size;
1128 igen6_pvt->ms_l_map = ms_l_map;
1129
1130 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n",
1131 ms_s_size >> 20, ms_l_map);
1132
1133 if (!size)
1134 return 0;
1135
1136 cmf = ioremap(base, size);
1137 if (!cmf) {
1138 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base);
1139 return -ENODEV;
1140 }
1141
1142 ms_hash = readq(cmf + offset);
1143 igen6_pvt->ms_hash = ms_hash;
1144
1145 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash);
1146
1147 iounmap(cmf);
1148
1149 return 0;
1150}
1151
1152static int register_err_handler(void)
1153{
1154 int rc;
1155
1156 if (res_cfg->machine_check) {
1157 mce_register_decode_chain(&ecclog_mce_dec);
1158 return 0;
1159 }
1160
1161 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler,
1162 0, IGEN6_NMI_NAME);
1163 if (rc) {
1164 igen6_printk(KERN_ERR, "Failed to register NMI handler\n");
1165 return rc;
1166 }
1167
1168 return 0;
1169}
1170
1171static void unregister_err_handler(void)
1172{
1173 if (res_cfg->machine_check) {
1174 mce_unregister_decode_chain(&ecclog_mce_dec);
1175 return;
1176 }
1177
1178 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1179}
1180
1181static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1182{
1183 u64 mchbar;
1184 int i, rc;
1185
1186 edac_dbg(2, "\n");
1187
1188 igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL);
1189 if (!igen6_pvt)
1190 return -ENOMEM;
1191
1192 res_cfg = (struct res_config *)ent->driver_data;
1193
1194 rc = igen6_pci_setup(pdev, &mchbar);
1195 if (rc)
1196 goto fail;
1197
1198 for (i = 0; i < res_cfg->num_imc; i++) {
1199 rc = igen6_register_mci(i, mchbar, pdev);
1200 if (rc)
1201 goto fail2;
1202 }
1203
1204 if (res_cfg->num_imc > 1) {
1205 rc = igen6_mem_slice_setup(mchbar);
1206 if (rc)
1207 goto fail2;
1208 }
1209
1210 ecclog_pool = ecclog_gen_pool_create();
1211 if (!ecclog_pool) {
1212 rc = -ENOMEM;
1213 goto fail2;
1214 }
1215
1216 INIT_WORK(&ecclog_work, ecclog_work_cb);
1217 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb);
1218
1219 /* Check if any pending errors before registering the NMI handler */
1220 ecclog_handler();
1221
1222 rc = register_err_handler();
1223 if (rc)
1224 goto fail3;
1225
1226 /* Enable error reporting */
1227 rc = errcmd_enable_error_reporting(true);
1228 if (rc) {
1229 igen6_printk(KERN_ERR, "Failed to enable error reporting\n");
1230 goto fail4;
1231 }
1232
1233 igen6_debug_setup();
1234 return 0;
1235fail4:
1236 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1237fail3:
1238 gen_pool_destroy(ecclog_pool);
1239fail2:
1240 igen6_unregister_mcis();
1241fail:
1242 kfree(igen6_pvt);
1243 return rc;
1244}
1245
1246static void igen6_remove(struct pci_dev *pdev)
1247{
1248 edac_dbg(2, "\n");
1249
1250 igen6_debug_teardown();
1251 errcmd_enable_error_reporting(false);
1252 unregister_err_handler();
1253 irq_work_sync(&ecclog_irq_work);
1254 flush_work(&ecclog_work);
1255 gen_pool_destroy(ecclog_pool);
1256 igen6_unregister_mcis();
1257 kfree(igen6_pvt);
1258}
1259
1260static struct pci_driver igen6_driver = {
1261 .name = EDAC_MOD_STR,
1262 .probe = igen6_probe,
1263 .remove = igen6_remove,
1264 .id_table = igen6_pci_tbl,
1265};
1266
1267static int __init igen6_init(void)
1268{
1269 const char *owner;
1270 int rc;
1271
1272 edac_dbg(2, "\n");
1273
1274 if (ghes_get_devices())
1275 return -EBUSY;
1276
1277 owner = edac_get_owner();
1278 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
1279 return -EBUSY;
1280
1281 edac_op_state = EDAC_OPSTATE_NMI;
1282
1283 rc = pci_register_driver(&igen6_driver);
1284 if (rc)
1285 return rc;
1286
1287 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION);
1288
1289 return 0;
1290}
1291
1292static void __exit igen6_exit(void)
1293{
1294 edac_dbg(2, "\n");
1295
1296 pci_unregister_driver(&igen6_driver);
1297}
1298
1299module_init(igen6_init);
1300module_exit(igen6_exit);
1301
1302MODULE_LICENSE("GPL v2");
1303MODULE_AUTHOR("Qiuxu Zhuo");
1304MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC");