1// SPDX-License-Identifier: GPL-2.0-only
   2/* Intel i7 core/Nehalem Memory Controller kernel module
   3 *
   4 * This driver supports the memory controllers found on the Intel
   5 * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   6 * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   7 * and Westmere-EP.
   8 *
   9 * Copyright (c) 2009-2010 by:
  10 *	 Mauro Carvalho Chehab
  11 *
  12 * Red Hat Inc. https://www.redhat.com
  13 *
  14 * Forked and adapted from the i5400_edac driver
  15 *
  16 * Based on the following public Intel datasheets:
  17 * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  18 * Datasheet, Volume 2:
  19 *	http://download.intel.com/design/processor/datashts/320835.pdf
  20 * Intel Xeon Processor 5500 Series Datasheet Volume 2
  21 *	http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  22 * also available at:
  23 * 	http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  24 */
  25
  26#include <linux/module.h>
  27#include <linux/init.h>
  28#include <linux/pci.h>
  29#include <linux/pci_ids.h>
  30#include <linux/slab.h>
  31#include <linux/delay.h>
  32#include <linux/dmi.h>
  33#include <linux/edac.h>
  34#include <linux/mmzone.h>
  35#include <linux/smp.h>
  36#include <asm/mce.h>
  37#include <asm/processor.h>
  38#include <asm/div64.h>
  39
  40#include "edac_module.h"
  41
  42/* Static vars */
  43static LIST_HEAD(i7core_edac_list);
  44static DEFINE_MUTEX(i7core_edac_lock);
  45static int probed;
  46
  47static int use_pci_fixup;
  48module_param(use_pci_fixup, int, 0444);
  49MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  50/*
  51 * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  52 * registers start at bus 255, and are not reported by BIOS.
  53 * We currently find devices with only 2 sockets. In order to support more QPI
  54 * Quick Path Interconnect, just increment this number.
  55 */
  56#define MAX_SOCKET_BUSES	2
  57
  58
  59/*
  60 * Alter this version for the module when modifications are made
  61 */
  62#define I7CORE_REVISION    " Ver: 1.0.0"
  63#define EDAC_MOD_STR      "i7core_edac"
  64
  65/*
  66 * Debug macros
  67 */
  68#define i7core_printk(level, fmt, arg...)			\
  69	edac_printk(level, "i7core", fmt, ##arg)
  70
  71#define i7core_mc_printk(mci, level, fmt, arg...)		\
  72	edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  73
  74/*
  75 * i7core Memory Controller Registers
  76 */
  77
  78	/* OFFSETS for Device 0 Function 0 */
  79
  80#define MC_CFG_CONTROL	0x90
  81  #define MC_CFG_UNLOCK		0x02
  82  #define MC_CFG_LOCK		0x00
  83
  84	/* OFFSETS for Device 3 Function 0 */
  85
  86#define MC_CONTROL	0x48
  87#define MC_STATUS	0x4c
  88#define MC_MAX_DOD	0x64
  89
  90/*
  91 * OFFSETS for Device 3 Function 4, as indicated on Xeon 5500 datasheet:
  92 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  93 */
  94
  95#define MC_TEST_ERR_RCV1	0x60
  96  #define DIMM2_COR_ERR(r)			((r) & 0x7fff)
  97
  98#define MC_TEST_ERR_RCV0	0x64
  99  #define DIMM1_COR_ERR(r)			(((r) >> 16) & 0x7fff)
 100  #define DIMM0_COR_ERR(r)			((r) & 0x7fff)
 101
 102/* OFFSETS for Device 3 Function 2, as indicated on Xeon 5500 datasheet */
 103#define MC_SSRCONTROL		0x48
 104  #define SSR_MODE_DISABLE	0x00
 105  #define SSR_MODE_ENABLE	0x01
 106  #define SSR_MODE_MASK		0x03
 107
 108#define MC_SCRUB_CONTROL	0x4c
 109  #define STARTSCRUB		(1 << 24)
 110  #define SCRUBINTERVAL_MASK    0xffffff
 111
 112#define MC_COR_ECC_CNT_0	0x80
 113#define MC_COR_ECC_CNT_1	0x84
 114#define MC_COR_ECC_CNT_2	0x88
 115#define MC_COR_ECC_CNT_3	0x8c
 116#define MC_COR_ECC_CNT_4	0x90
 117#define MC_COR_ECC_CNT_5	0x94
 118
 119#define DIMM_TOP_COR_ERR(r)			(((r) >> 16) & 0x7fff)
 120#define DIMM_BOT_COR_ERR(r)			((r) & 0x7fff)
 121
 122
 123	/* OFFSETS for Devices 4,5 and 6 Function 0 */
 124
 125#define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 126  #define THREE_DIMMS_PRESENT		(1 << 24)
 127  #define SINGLE_QUAD_RANK_PRESENT	(1 << 23)
 128  #define QUAD_RANK_PRESENT		(1 << 22)
 129  #define REGISTERED_DIMM		(1 << 15)
 130
 131#define MC_CHANNEL_MAPPER	0x60
 132  #define RDLCH(r, ch)		((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 133  #define WRLCH(r, ch)		((((r) >> (ch * 6)) & 0x07) - 1)
 134
 135#define MC_CHANNEL_RANK_PRESENT 0x7c
 136  #define RANK_PRESENT_MASK		0xffff
 137
 138#define MC_CHANNEL_ADDR_MATCH	0xf0
 139#define MC_CHANNEL_ERROR_MASK	0xf8
 140#define MC_CHANNEL_ERROR_INJECT	0xfc
 141  #define INJECT_ADDR_PARITY	0x10
 142  #define INJECT_ECC		0x08
 143  #define MASK_CACHELINE	0x06
 144  #define MASK_FULL_CACHELINE	0x06
 145  #define MASK_MSB32_CACHELINE	0x04
 146  #define MASK_LSB32_CACHELINE	0x02
 147  #define NO_MASK_CACHELINE	0x00
 148  #define REPEAT_EN		0x01
 149
 150	/* OFFSETS for Devices 4,5 and 6 Function 1 */
 151
 152#define MC_DOD_CH_DIMM0		0x48
 153#define MC_DOD_CH_DIMM1		0x4c
 154#define MC_DOD_CH_DIMM2		0x50
 155  #define RANKOFFSET_MASK	((1 << 12) | (1 << 11) | (1 << 10))
 156  #define RANKOFFSET(x)		((x & RANKOFFSET_MASK) >> 10)
 157  #define DIMM_PRESENT_MASK	(1 << 9)
 158  #define DIMM_PRESENT(x)	(((x) & DIMM_PRESENT_MASK) >> 9)
 159  #define MC_DOD_NUMBANK_MASK		((1 << 8) | (1 << 7))
 160  #define MC_DOD_NUMBANK(x)		(((x) & MC_DOD_NUMBANK_MASK) >> 7)
 161  #define MC_DOD_NUMRANK_MASK		((1 << 6) | (1 << 5))
 162  #define MC_DOD_NUMRANK(x)		(((x) & MC_DOD_NUMRANK_MASK) >> 5)
 163  #define MC_DOD_NUMROW_MASK		((1 << 4) | (1 << 3) | (1 << 2))
 164  #define MC_DOD_NUMROW(x)		(((x) & MC_DOD_NUMROW_MASK) >> 2)
 165  #define MC_DOD_NUMCOL_MASK		3
 166  #define MC_DOD_NUMCOL(x)		((x) & MC_DOD_NUMCOL_MASK)
 167
 168#define MC_RANK_PRESENT		0x7c
 169
 170#define MC_SAG_CH_0	0x80
 171#define MC_SAG_CH_1	0x84
 172#define MC_SAG_CH_2	0x88
 173#define MC_SAG_CH_3	0x8c
 174#define MC_SAG_CH_4	0x90
 175#define MC_SAG_CH_5	0x94
 176#define MC_SAG_CH_6	0x98
 177#define MC_SAG_CH_7	0x9c
 178
 179#define MC_RIR_LIMIT_CH_0	0x40
 180#define MC_RIR_LIMIT_CH_1	0x44
 181#define MC_RIR_LIMIT_CH_2	0x48
 182#define MC_RIR_LIMIT_CH_3	0x4C
 183#define MC_RIR_LIMIT_CH_4	0x50
 184#define MC_RIR_LIMIT_CH_5	0x54
 185#define MC_RIR_LIMIT_CH_6	0x58
 186#define MC_RIR_LIMIT_CH_7	0x5C
 187#define MC_RIR_LIMIT_MASK	((1 << 10) - 1)
 188
 189#define MC_RIR_WAY_CH		0x80
 190  #define MC_RIR_WAY_OFFSET_MASK	(((1 << 14) - 1) & ~0x7)
 191  #define MC_RIR_WAY_RANK_MASK		0x7
 192
 193/*
 194 * i7core structs
 195 */
 196
 197#define NUM_CHANS 3
 198#define MAX_DIMMS 3		/* Max DIMMS per channel */
 199#define MAX_MCR_FUNC  4
 200#define MAX_CHAN_FUNC 3
 201
 202struct i7core_info {
 203	u32	mc_control;
 204	u32	mc_status;
 205	u32	max_dod;
 206	u32	ch_map;
 207};
 208
 209
 210struct i7core_inject {
 211	int	enable;
 212
 213	u32	section;
 214	u32	type;
 215	u32	eccmask;
 216
 217	/* Error address mask */
 218	int channel, dimm, rank, bank, page, col;
 219};
 220
 221struct i7core_channel {
 222	bool		is_3dimms_present;
 223	bool		is_single_4rank;
 224	bool		has_4rank;
 225	u32		dimms;
 226};
 227
 228struct pci_id_descr {
 229	int			dev;
 230	int			func;
 231	int 			dev_id;
 232	int			optional;
 233};
 234
 235struct pci_id_table {
 236	const struct pci_id_descr	*descr;
 237	int				n_devs;
 238};
 239
 240struct i7core_dev {
 241	struct list_head	list;
 242	u8			socket;
 243	struct pci_dev		**pdev;
 244	int			n_devs;
 245	struct mem_ctl_info	*mci;
 246};
 247
 248struct i7core_pvt {
 249	struct device *addrmatch_dev, *chancounts_dev;
 250
 251	struct pci_dev	*pci_noncore;
 252	struct pci_dev	*pci_mcr[MAX_MCR_FUNC + 1];
 253	struct pci_dev	*pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 254
 255	struct i7core_dev *i7core_dev;
 256
 257	struct i7core_info	info;
 258	struct i7core_inject	inject;
 259	struct i7core_channel	channel[NUM_CHANS];
 260
 261	int		ce_count_available;
 262
 263			/* ECC corrected errors counts per udimm */
 264	unsigned long	udimm_ce_count[MAX_DIMMS];
 265	int		udimm_last_ce_count[MAX_DIMMS];
 266			/* ECC corrected errors counts per rdimm */
 267	unsigned long	rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 268	int		rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 269
 270	bool		is_registered, enable_scrub;
 271
 272	/* DCLK Frequency used for computing scrub rate */
 273	int			dclk_freq;
 274
 275	/* Struct to control EDAC polling */
 276	struct edac_pci_ctl_info *i7core_pci;
 277};
 278
 279#define PCI_DESCR(device, function, device_id)	\
 280	.dev = (device),			\
 281	.func = (function),			\
 282	.dev_id = (device_id)
 283
 284static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 285		/* Memory controller */
 286	{ PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 287	{ PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 288			/* Exists only for RDIMM */
 289	{ PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 290	{ PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 291
 292		/* Channel 0 */
 293	{ PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 294	{ PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 295	{ PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 296	{ PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 297
 298		/* Channel 1 */
 299	{ PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 300	{ PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 301	{ PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 302	{ PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 303
 304		/* Channel 2 */
 305	{ PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 306	{ PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 307	{ PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 308	{ PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 309
 310		/* Generic Non-core registers */
 311	/*
 312	 * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 313	 * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 314	 * the probing code needs to test for the other address in case of
 315	 * failure of this one
 316	 */
 317	{ PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 318
 319};
 320
 321static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 322	{ PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 323	{ PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 324	{ PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 325
 326	{ PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 327	{ PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 328	{ PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 329	{ PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 330
 331	{ PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 332	{ PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 333	{ PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 334	{ PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 335
 336	/*
 337	 * This is the PCI device has an alternate address on some
 338	 * processors like Core i7 860
 339	 */
 340	{ PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 341};
 342
 343static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 344		/* Memory controller */
 345	{ PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 346	{ PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 347			/* Exists only for RDIMM */
 348	{ PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 349	{ PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 350
 351		/* Channel 0 */
 352	{ PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 353	{ PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 354	{ PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 355	{ PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 356
 357		/* Channel 1 */
 358	{ PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 359	{ PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 360	{ PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 361	{ PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 362
 363		/* Channel 2 */
 364	{ PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 365	{ PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 366	{ PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 367	{ PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 368
 369		/* Generic Non-core registers */
 370	{ PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 371
 372};
 373
 374#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 375static const struct pci_id_table pci_dev_table[] = {
 376	PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 377	PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 378	PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 379	{ NULL, }
 380};
 381
 382/*
 383 *	pci_device_id	table for which devices we are looking for
 384 */
 385static const struct pci_device_id i7core_pci_tbl[] = {
 386	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 387	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 388	{ 0, }
 389};
 390
 391/****************************************************************************
 392			Ancillary status routines
 393 ****************************************************************************/
 394
 395	/* MC_CONTROL bits */
 396#define CH_ACTIVE(pvt, ch)	((pvt)->info.mc_control & (1 << (8 + ch)))
 397#define ECCx8(pvt)		((pvt)->info.mc_control & (1 << 1))
 398
 399	/* MC_STATUS bits */
 400#define ECC_ENABLED(pvt)	((pvt)->info.mc_status & (1 << 4))
 401#define CH_DISABLED(pvt, ch)	((pvt)->info.mc_status & (1 << ch))
 402
 403	/* MC_MAX_DOD read functions */
 404static inline int numdimms(u32 dimms)
 405{
 406	return (dimms & 0x3) + 1;
 407}
 408
 409static inline int numrank(u32 rank)
 410{
 411	static const int ranks[] = { 1, 2, 4, -EINVAL };
 412
 413	return ranks[rank & 0x3];
 414}
 415
 416static inline int numbank(u32 bank)
 417{
 418	static const int banks[] = { 4, 8, 16, -EINVAL };
 419
 420	return banks[bank & 0x3];
 421}
 422
 423static inline int numrow(u32 row)
 424{
 425	static const int rows[] = {
 426		1 << 12, 1 << 13, 1 << 14, 1 << 15,
 427		1 << 16, -EINVAL, -EINVAL, -EINVAL,
 428	};
 429
 430	return rows[row & 0x7];
 431}
 432
 433static inline int numcol(u32 col)
 434{
 435	static const int cols[] = {
 436		1 << 10, 1 << 11, 1 << 12, -EINVAL,
 437	};
 438	return cols[col & 0x3];
 439}
 440
 441static struct i7core_dev *get_i7core_dev(u8 socket)
 442{
 443	struct i7core_dev *i7core_dev;
 444
 445	list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 446		if (i7core_dev->socket == socket)
 447			return i7core_dev;
 448	}
 449
 450	return NULL;
 451}
 452
 453static struct i7core_dev *alloc_i7core_dev(u8 socket,
 454					   const struct pci_id_table *table)
 455{
 456	struct i7core_dev *i7core_dev;
 457
 458	i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 459	if (!i7core_dev)
 460		return NULL;
 461
 462	i7core_dev->pdev = kcalloc(table->n_devs, sizeof(*i7core_dev->pdev),
 463				   GFP_KERNEL);
 464	if (!i7core_dev->pdev) {
 465		kfree(i7core_dev);
 466		return NULL;
 467	}
 468
 469	i7core_dev->socket = socket;
 470	i7core_dev->n_devs = table->n_devs;
 471	list_add_tail(&i7core_dev->list, &i7core_edac_list);
 472
 473	return i7core_dev;
 474}
 475
 476static void free_i7core_dev(struct i7core_dev *i7core_dev)
 477{
 478	list_del(&i7core_dev->list);
 479	kfree(i7core_dev->pdev);
 480	kfree(i7core_dev);
 481}
 482
 483/****************************************************************************
 484			Memory check routines
 485 ****************************************************************************/
 486
 487static int get_dimm_config(struct mem_ctl_info *mci)
 488{
 489	struct i7core_pvt *pvt = mci->pvt_info;
 490	struct pci_dev *pdev;
 491	int i, j;
 492	enum edac_type mode;
 493	enum mem_type mtype;
 494	struct dimm_info *dimm;
 495
 496	/* Get data from the MC register, function 0 */
 497	pdev = pvt->pci_mcr[0];
 498	if (!pdev)
 499		return -ENODEV;
 500
 501	/* Device 3 function 0 reads */
 502	pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 503	pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 504	pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 505	pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 506
 507	edac_dbg(0, "QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 508		 pvt->i7core_dev->socket, pvt->info.mc_control,
 509		 pvt->info.mc_status, pvt->info.max_dod, pvt->info.ch_map);
 510
 511	if (ECC_ENABLED(pvt)) {
 512		edac_dbg(0, "ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 513		if (ECCx8(pvt))
 514			mode = EDAC_S8ECD8ED;
 515		else
 516			mode = EDAC_S4ECD4ED;
 517	} else {
 518		edac_dbg(0, "ECC disabled\n");
 519		mode = EDAC_NONE;
 520	}
 521
 522	/* FIXME: need to handle the error codes */
 523	edac_dbg(0, "DOD Max limits: DIMMS: %d, %d-ranked, %d-banked x%x x 0x%x\n",
 524		 numdimms(pvt->info.max_dod),
 525		 numrank(pvt->info.max_dod >> 2),
 526		 numbank(pvt->info.max_dod >> 4),
 527		 numrow(pvt->info.max_dod >> 6),
 528		 numcol(pvt->info.max_dod >> 9));
 529
 530	for (i = 0; i < NUM_CHANS; i++) {
 531		u32 data, dimm_dod[3], value[8];
 532
 533		if (!pvt->pci_ch[i][0])
 534			continue;
 535
 536		if (!CH_ACTIVE(pvt, i)) {
 537			edac_dbg(0, "Channel %i is not active\n", i);
 538			continue;
 539		}
 540		if (CH_DISABLED(pvt, i)) {
 541			edac_dbg(0, "Channel %i is disabled\n", i);
 542			continue;
 543		}
 544
 545		/* Devices 4-6 function 0 */
 546		pci_read_config_dword(pvt->pci_ch[i][0],
 547				MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 548
 549
 550		if (data & THREE_DIMMS_PRESENT)
 551			pvt->channel[i].is_3dimms_present = true;
 552
 553		if (data & SINGLE_QUAD_RANK_PRESENT)
 554			pvt->channel[i].is_single_4rank = true;
 555
 556		if (data & QUAD_RANK_PRESENT)
 557			pvt->channel[i].has_4rank = true;
 558
 559		if (data & REGISTERED_DIMM)
 560			mtype = MEM_RDDR3;
 561		else
 562			mtype = MEM_DDR3;
 563
 564		/* Devices 4-6 function 1 */
 565		pci_read_config_dword(pvt->pci_ch[i][1],
 566				MC_DOD_CH_DIMM0, &dimm_dod[0]);
 567		pci_read_config_dword(pvt->pci_ch[i][1],
 568				MC_DOD_CH_DIMM1, &dimm_dod[1]);
 569		pci_read_config_dword(pvt->pci_ch[i][1],
 570				MC_DOD_CH_DIMM2, &dimm_dod[2]);
 571
 572		edac_dbg(0, "Ch%d phy rd%d, wr%d (0x%08x): %s%s%s%cDIMMs\n",
 573			 i,
 574			 RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 575			 data,
 576			 pvt->channel[i].is_3dimms_present ? "3DIMMS " : "",
 577			 pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "",
 578			 pvt->channel[i].has_4rank ? "HAS_4R " : "",
 579			 (data & REGISTERED_DIMM) ? 'R' : 'U');
 580
 581		for (j = 0; j < 3; j++) {
 582			u32 banks, ranks, rows, cols;
 583			u32 size, npages;
 584
 585			if (!DIMM_PRESENT(dimm_dod[j]))
 586				continue;
 587
 588			dimm = edac_get_dimm(mci, i, j, 0);
 589			banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 590			ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 591			rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 592			cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 593
 594			/* DDR3 has 8 I/O banks */
 595			size = (rows * cols * banks * ranks) >> (20 - 3);
 596
 597			edac_dbg(0, "\tdimm %d %d MiB offset: %x, bank: %d, rank: %d, row: %#x, col: %#x\n",
 598				 j, size,
 599				 RANKOFFSET(dimm_dod[j]),
 600				 banks, ranks, rows, cols);
 601
 602			npages = MiB_TO_PAGES(size);
 603
 604			dimm->nr_pages = npages;
 605
 606			switch (banks) {
 607			case 4:
 608				dimm->dtype = DEV_X4;
 609				break;
 610			case 8:
 611				dimm->dtype = DEV_X8;
 612				break;
 613			case 16:
 614				dimm->dtype = DEV_X16;
 615				break;
 616			default:
 617				dimm->dtype = DEV_UNKNOWN;
 618			}
 619
 620			snprintf(dimm->label, sizeof(dimm->label),
 621				 "CPU#%uChannel#%u_DIMM#%u",
 622				 pvt->i7core_dev->socket, i, j);
 623			dimm->grain = 8;
 624			dimm->edac_mode = mode;
 625			dimm->mtype = mtype;
 626		}
 627
 628		pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 629		pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 630		pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 631		pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 632		pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 633		pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 634		pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 635		pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 636		edac_dbg(1, "\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 637		for (j = 0; j < 8; j++)
 638			edac_dbg(1, "\t\t%#x\t%#x\t%#x\n",
 639				 (value[j] >> 27) & 0x1,
 640				 (value[j] >> 24) & 0x7,
 641				 (value[j] & ((1 << 24) - 1)));
 642	}
 643
 644	return 0;
 645}
 646
 647/****************************************************************************
 648			Error insertion routines
 649 ****************************************************************************/
 650
 651#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
 652
 653/* The i7core has independent error injection features per channel.
 654   However, to have a simpler code, we don't allow enabling error injection
 655   on more than one channel.
 656   Also, since a change at an inject parameter will be applied only at enable,
 657   we're disabling error injection on all write calls to the sysfs nodes that
 658   controls the error code injection.
 659 */
 660static int disable_inject(const struct mem_ctl_info *mci)
 661{
 662	struct i7core_pvt *pvt = mci->pvt_info;
 663
 664	pvt->inject.enable = 0;
 665
 666	if (!pvt->pci_ch[pvt->inject.channel][0])
 667		return -ENODEV;
 668
 669	pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 670				MC_CHANNEL_ERROR_INJECT, 0);
 671
 672	return 0;
 673}
 674
 675/*
 676 * i7core inject inject.section
 677 *
 678 *	accept and store error injection inject.section value
 679 *	bit 0 - refers to the lower 32-byte half cacheline
 680 *	bit 1 - refers to the upper 32-byte half cacheline
 681 */
 682static ssize_t i7core_inject_section_store(struct device *dev,
 683					   struct device_attribute *mattr,
 684					   const char *data, size_t count)
 685{
 686	struct mem_ctl_info *mci = to_mci(dev);
 687	struct i7core_pvt *pvt = mci->pvt_info;
 688	unsigned long value;
 689	int rc;
 690
 691	if (pvt->inject.enable)
 692		disable_inject(mci);
 693
 694	rc = kstrtoul(data, 10, &value);
 695	if ((rc < 0) || (value > 3))
 696		return -EIO;
 697
 698	pvt->inject.section = (u32) value;
 699	return count;
 700}
 701
 702static ssize_t i7core_inject_section_show(struct device *dev,
 703					  struct device_attribute *mattr,
 704					  char *data)
 705{
 706	struct mem_ctl_info *mci = to_mci(dev);
 707	struct i7core_pvt *pvt = mci->pvt_info;
 708	return sprintf(data, "0x%08x\n", pvt->inject.section);
 709}
 710
 711/*
 712 * i7core inject.type
 713 *
 714 *	accept and store error injection inject.section value
 715 *	bit 0 - repeat enable - Enable error repetition
 716 *	bit 1 - inject ECC error
 717 *	bit 2 - inject parity error
 718 */
 719static ssize_t i7core_inject_type_store(struct device *dev,
 720					struct device_attribute *mattr,
 721					const char *data, size_t count)
 722{
 723	struct mem_ctl_info *mci = to_mci(dev);
 724	struct i7core_pvt *pvt = mci->pvt_info;
 725	unsigned long value;
 726	int rc;
 727
 728	if (pvt->inject.enable)
 729		disable_inject(mci);
 730
 731	rc = kstrtoul(data, 10, &value);
 732	if ((rc < 0) || (value > 7))
 733		return -EIO;
 734
 735	pvt->inject.type = (u32) value;
 736	return count;
 737}
 738
 739static ssize_t i7core_inject_type_show(struct device *dev,
 740				       struct device_attribute *mattr,
 741				       char *data)
 742{
 743	struct mem_ctl_info *mci = to_mci(dev);
 744	struct i7core_pvt *pvt = mci->pvt_info;
 745
 746	return sprintf(data, "0x%08x\n", pvt->inject.type);
 747}
 748
 749/*
 750 * i7core_inject_inject.eccmask_store
 751 *
 752 * The type of error (UE/CE) will depend on the inject.eccmask value:
 753 *   Any bits set to a 1 will flip the corresponding ECC bit
 754 *   Correctable errors can be injected by flipping 1 bit or the bits within
 755 *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 756 *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 757 *   uncorrectable error to be injected.
 758 */
 759static ssize_t i7core_inject_eccmask_store(struct device *dev,
 760					   struct device_attribute *mattr,
 761					   const char *data, size_t count)
 762{
 763	struct mem_ctl_info *mci = to_mci(dev);
 764	struct i7core_pvt *pvt = mci->pvt_info;
 765	unsigned long value;
 766	int rc;
 767
 768	if (pvt->inject.enable)
 769		disable_inject(mci);
 770
 771	rc = kstrtoul(data, 10, &value);
 772	if (rc < 0)
 773		return -EIO;
 774
 775	pvt->inject.eccmask = (u32) value;
 776	return count;
 777}
 778
 779static ssize_t i7core_inject_eccmask_show(struct device *dev,
 780					  struct device_attribute *mattr,
 781					  char *data)
 782{
 783	struct mem_ctl_info *mci = to_mci(dev);
 784	struct i7core_pvt *pvt = mci->pvt_info;
 785
 786	return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 787}
 788
 789/*
 790 * i7core_addrmatch
 791 *
 792 * The type of error (UE/CE) will depend on the inject.eccmask value:
 793 *   Any bits set to a 1 will flip the corresponding ECC bit
 794 *   Correctable errors can be injected by flipping 1 bit or the bits within
 795 *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 796 *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 797 *   uncorrectable error to be injected.
 798 */
 799
 800#define DECLARE_ADDR_MATCH(param, limit)			\
 801static ssize_t i7core_inject_store_##param(			\
 802	struct device *dev,					\
 803	struct device_attribute *mattr,				\
 804	const char *data, size_t count)				\
 805{								\
 806	struct mem_ctl_info *mci = dev_get_drvdata(dev);	\
 807	struct i7core_pvt *pvt;					\
 808	long value;						\
 809	int rc;							\
 810								\
 811	edac_dbg(1, "\n");					\
 812	pvt = mci->pvt_info;					\
 813								\
 814	if (pvt->inject.enable)					\
 815		disable_inject(mci);				\
 816								\
 817	if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 818		value = -1;					\
 819	else {							\
 820		rc = kstrtoul(data, 10, &value);		\
 821		if ((rc < 0) || (value >= limit))		\
 822			return -EIO;				\
 823	}							\
 824								\
 825	pvt->inject.param = value;				\
 826								\
 827	return count;						\
 828}								\
 829								\
 830static ssize_t i7core_inject_show_##param(			\
 831	struct device *dev,					\
 832	struct device_attribute *mattr,				\
 833	char *data)						\
 834{								\
 835	struct mem_ctl_info *mci = dev_get_drvdata(dev);	\
 836	struct i7core_pvt *pvt;					\
 837								\
 838	pvt = mci->pvt_info;					\
 839	edac_dbg(1, "pvt=%p\n", pvt);				\
 840	if (pvt->inject.param < 0)				\
 841		return sprintf(data, "any\n");			\
 842	else							\
 843		return sprintf(data, "%d\n", pvt->inject.param);\
 844}
 845
 846#define ATTR_ADDR_MATCH(param)					\
 847	static DEVICE_ATTR(param, S_IRUGO | S_IWUSR,		\
 848		    i7core_inject_show_##param,			\
 849		    i7core_inject_store_##param)
 850
 851DECLARE_ADDR_MATCH(channel, 3);
 852DECLARE_ADDR_MATCH(dimm, 3);
 853DECLARE_ADDR_MATCH(rank, 4);
 854DECLARE_ADDR_MATCH(bank, 32);
 855DECLARE_ADDR_MATCH(page, 0x10000);
 856DECLARE_ADDR_MATCH(col, 0x4000);
 857
 858ATTR_ADDR_MATCH(channel);
 859ATTR_ADDR_MATCH(dimm);
 860ATTR_ADDR_MATCH(rank);
 861ATTR_ADDR_MATCH(bank);
 862ATTR_ADDR_MATCH(page);
 863ATTR_ADDR_MATCH(col);
 864
 865static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 866{
 867	u32 read;
 868	int count;
 869
 870	edac_dbg(0, "setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 871		 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 872		 where, val);
 873
 874	for (count = 0; count < 10; count++) {
 875		if (count)
 876			msleep(100);
 877		pci_write_config_dword(dev, where, val);
 878		pci_read_config_dword(dev, where, &read);
 879
 880		if (read == val)
 881			return 0;
 882	}
 883
 884	i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 885		"write=%08x. Read=%08x\n",
 886		dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 887		where, val, read);
 888
 889	return -EINVAL;
 890}
 891
 892/*
 893 * This routine prepares the Memory Controller for error injection.
 894 * The error will be injected when some process tries to write to the
 895 * memory that matches the given criteria.
 896 * The criteria can be set in terms of a mask where dimm, rank, bank, page
 897 * and col can be specified.
 898 * A -1 value for any of the mask items will make the MCU to ignore
 899 * that matching criteria for error injection.
 900 *
 901 * It should be noticed that the error will only happen after a write operation
 902 * on a memory that matches the condition. if REPEAT_EN is not enabled at
 903 * inject mask, then it will produce just one error. Otherwise, it will repeat
 904 * until the injectmask would be cleaned.
 905 *
 906 * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 907 *    is reliable enough to check if the MC is using the
 908 *    three channels. However, this is not clear at the datasheet.
 909 */
 910static ssize_t i7core_inject_enable_store(struct device *dev,
 911					  struct device_attribute *mattr,
 912					  const char *data, size_t count)
 913{
 914	struct mem_ctl_info *mci = to_mci(dev);
 915	struct i7core_pvt *pvt = mci->pvt_info;
 916	u32 injectmask;
 917	u64 mask = 0;
 918	int  rc;
 919	long enable;
 920
 921	if (!pvt->pci_ch[pvt->inject.channel][0])
 922		return 0;
 923
 924	rc = kstrtoul(data, 10, &enable);
 925	if ((rc < 0))
 926		return 0;
 927
 928	if (enable) {
 929		pvt->inject.enable = 1;
 930	} else {
 931		disable_inject(mci);
 932		return count;
 933	}
 934
 935	/* Sets pvt->inject.dimm mask */
 936	if (pvt->inject.dimm < 0)
 937		mask |= 1LL << 41;
 938	else {
 939		if (pvt->channel[pvt->inject.channel].dimms > 2)
 940			mask |= (pvt->inject.dimm & 0x3LL) << 35;
 941		else
 942			mask |= (pvt->inject.dimm & 0x1LL) << 36;
 943	}
 944
 945	/* Sets pvt->inject.rank mask */
 946	if (pvt->inject.rank < 0)
 947		mask |= 1LL << 40;
 948	else {
 949		if (pvt->channel[pvt->inject.channel].dimms > 2)
 950			mask |= (pvt->inject.rank & 0x1LL) << 34;
 951		else
 952			mask |= (pvt->inject.rank & 0x3LL) << 34;
 953	}
 954
 955	/* Sets pvt->inject.bank mask */
 956	if (pvt->inject.bank < 0)
 957		mask |= 1LL << 39;
 958	else
 959		mask |= (pvt->inject.bank & 0x15LL) << 30;
 960
 961	/* Sets pvt->inject.page mask */
 962	if (pvt->inject.page < 0)
 963		mask |= 1LL << 38;
 964	else
 965		mask |= (pvt->inject.page & 0xffff) << 14;
 966
 967	/* Sets pvt->inject.column mask */
 968	if (pvt->inject.col < 0)
 969		mask |= 1LL << 37;
 970	else
 971		mask |= (pvt->inject.col & 0x3fff);
 972
 973	/*
 974	 * bit    0: REPEAT_EN
 975	 * bits 1-2: MASK_HALF_CACHELINE
 976	 * bit    3: INJECT_ECC
 977	 * bit    4: INJECT_ADDR_PARITY
 978	 */
 979
 980	injectmask = (pvt->inject.type & 1) |
 981		     (pvt->inject.section & 0x3) << 1 |
 982		     (pvt->inject.type & 0x6) << (3 - 1);
 983
 984	/* Unlock writes to registers - this register is write only */
 985	pci_write_config_dword(pvt->pci_noncore,
 986			       MC_CFG_CONTROL, 0x2);
 987
 988	write_and_test(pvt->pci_ch[pvt->inject.channel][0],
 989			       MC_CHANNEL_ADDR_MATCH, mask);
 990	write_and_test(pvt->pci_ch[pvt->inject.channel][0],
 991			       MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
 992
 993	write_and_test(pvt->pci_ch[pvt->inject.channel][0],
 994			       MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
 995
 996	write_and_test(pvt->pci_ch[pvt->inject.channel][0],
 997			       MC_CHANNEL_ERROR_INJECT, injectmask);
 998
 999	/*
1000	 * This is something undocumented, based on my tests
1001	 * Without writing 8 to this register, errors aren't injected. Not sure
1002	 * why.
1003	 */
1004	pci_write_config_dword(pvt->pci_noncore,
1005			       MC_CFG_CONTROL, 8);
1006
1007	edac_dbg(0, "Error inject addr match 0x%016llx, ecc 0x%08x, inject 0x%08x\n",
1008		 mask, pvt->inject.eccmask, injectmask);
1009
1010
1011	return count;
1012}
1013
1014static ssize_t i7core_inject_enable_show(struct device *dev,
1015					 struct device_attribute *mattr,
1016					 char *data)
1017{
1018	struct mem_ctl_info *mci = to_mci(dev);
1019	struct i7core_pvt *pvt = mci->pvt_info;
1020	u32 injectmask;
1021
1022	if (!pvt->pci_ch[pvt->inject.channel][0])
1023		return 0;
1024
1025	pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1026			       MC_CHANNEL_ERROR_INJECT, &injectmask);
1027
1028	edac_dbg(0, "Inject error read: 0x%018x\n", injectmask);
1029
1030	if (injectmask & 0x0c)
1031		pvt->inject.enable = 1;
1032
1033	return sprintf(data, "%d\n", pvt->inject.enable);
1034}
1035
1036#define DECLARE_COUNTER(param)					\
1037static ssize_t i7core_show_counter_##param(			\
1038	struct device *dev,					\
1039	struct device_attribute *mattr,				\
1040	char *data)						\
1041{								\
1042	struct mem_ctl_info *mci = dev_get_drvdata(dev);	\
1043	struct i7core_pvt *pvt = mci->pvt_info;			\
1044								\
1045	edac_dbg(1, "\n");					\
1046	if (!pvt->ce_count_available || (pvt->is_registered))	\
1047		return sprintf(data, "data unavailable\n");	\
1048	return sprintf(data, "%lu\n",				\
1049			pvt->udimm_ce_count[param]);		\
1050}
1051
1052#define ATTR_COUNTER(param)					\
1053	static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR,	\
1054		    i7core_show_counter_##param,		\
1055		    NULL)
1056
1057DECLARE_COUNTER(0);
1058DECLARE_COUNTER(1);
1059DECLARE_COUNTER(2);
1060
1061ATTR_COUNTER(0);
1062ATTR_COUNTER(1);
1063ATTR_COUNTER(2);
1064
1065/*
1066 * inject_addrmatch device sysfs struct
1067 */
1068
1069static struct attribute *i7core_addrmatch_attrs[] = {
1070	&dev_attr_channel.attr,
1071	&dev_attr_dimm.attr,
1072	&dev_attr_rank.attr,
1073	&dev_attr_bank.attr,
1074	&dev_attr_page.attr,
1075	&dev_attr_col.attr,
1076	NULL
1077};
1078
1079static const struct attribute_group addrmatch_grp = {
1080	.attrs	= i7core_addrmatch_attrs,
1081};
1082
1083static const struct attribute_group *addrmatch_groups[] = {
1084	&addrmatch_grp,
1085	NULL
1086};
1087
1088static void addrmatch_release(struct device *device)
1089{
1090	edac_dbg(1, "Releasing device %s\n", dev_name(device));
1091	kfree(device);
1092}
1093
1094static const struct device_type addrmatch_type = {
1095	.groups		= addrmatch_groups,
1096	.release	= addrmatch_release,
1097};
1098
1099/*
1100 * all_channel_counts sysfs struct
1101 */
1102
1103static struct attribute *i7core_udimm_counters_attrs[] = {
1104	&dev_attr_udimm0.attr,
1105	&dev_attr_udimm1.attr,
1106	&dev_attr_udimm2.attr,
1107	NULL
1108};
1109
1110static const struct attribute_group all_channel_counts_grp = {
1111	.attrs	= i7core_udimm_counters_attrs,
1112};
1113
1114static const struct attribute_group *all_channel_counts_groups[] = {
1115	&all_channel_counts_grp,
1116	NULL
1117};
1118
1119static void all_channel_counts_release(struct device *device)
1120{
1121	edac_dbg(1, "Releasing device %s\n", dev_name(device));
1122	kfree(device);
1123}
1124
1125static const struct device_type all_channel_counts_type = {
1126	.groups		= all_channel_counts_groups,
1127	.release	= all_channel_counts_release,
1128};
1129
1130/*
1131 * inject sysfs attributes
1132 */
1133
1134static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
1135		   i7core_inject_section_show, i7core_inject_section_store);
1136
1137static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR,
1138		   i7core_inject_type_show, i7core_inject_type_store);
1139
1140
1141static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR,
1142		   i7core_inject_eccmask_show, i7core_inject_eccmask_store);
1143
1144static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR,
1145		   i7core_inject_enable_show, i7core_inject_enable_store);
1146
1147static struct attribute *i7core_dev_attrs[] = {
1148	&dev_attr_inject_section.attr,
1149	&dev_attr_inject_type.attr,
1150	&dev_attr_inject_eccmask.attr,
1151	&dev_attr_inject_enable.attr,
1152	NULL
1153};
1154
1155ATTRIBUTE_GROUPS(i7core_dev);
1156
1157static int i7core_create_sysfs_devices(struct mem_ctl_info *mci)
1158{
1159	struct i7core_pvt *pvt = mci->pvt_info;
1160	int rc;
1161
1162	pvt->addrmatch_dev = kzalloc(sizeof(*pvt->addrmatch_dev), GFP_KERNEL);
1163	if (!pvt->addrmatch_dev)
1164		return -ENOMEM;
1165
1166	pvt->addrmatch_dev->type = &addrmatch_type;
1167	pvt->addrmatch_dev->bus = mci->dev.bus;
1168	device_initialize(pvt->addrmatch_dev);
1169	pvt->addrmatch_dev->parent = &mci->dev;
1170	dev_set_name(pvt->addrmatch_dev, "inject_addrmatch");
1171	dev_set_drvdata(pvt->addrmatch_dev, mci);
1172
1173	edac_dbg(1, "creating %s\n", dev_name(pvt->addrmatch_dev));
1174
1175	rc = device_add(pvt->addrmatch_dev);
1176	if (rc < 0)
1177		goto err_put_addrmatch;
1178
1179	if (!pvt->is_registered) {
1180		pvt->chancounts_dev = kzalloc(sizeof(*pvt->chancounts_dev),
1181					      GFP_KERNEL);
1182		if (!pvt->chancounts_dev) {
1183			rc = -ENOMEM;
1184			goto err_del_addrmatch;
1185		}
1186
1187		pvt->chancounts_dev->type = &all_channel_counts_type;
1188		pvt->chancounts_dev->bus = mci->dev.bus;
1189		device_initialize(pvt->chancounts_dev);
1190		pvt->chancounts_dev->parent = &mci->dev;
1191		dev_set_name(pvt->chancounts_dev, "all_channel_counts");
1192		dev_set_drvdata(pvt->chancounts_dev, mci);
1193
1194		edac_dbg(1, "creating %s\n", dev_name(pvt->chancounts_dev));
1195
1196		rc = device_add(pvt->chancounts_dev);
1197		if (rc < 0)
1198			goto err_put_chancounts;
1199	}
1200	return 0;
1201
1202err_put_chancounts:
1203	put_device(pvt->chancounts_dev);
1204err_del_addrmatch:
1205	device_del(pvt->addrmatch_dev);
1206err_put_addrmatch:
1207	put_device(pvt->addrmatch_dev);
1208
1209	return rc;
1210}
1211
1212static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci)
1213{
1214	struct i7core_pvt *pvt = mci->pvt_info;
1215
1216	edac_dbg(1, "\n");
1217
1218	if (!pvt->is_registered) {
1219		device_del(pvt->chancounts_dev);
1220		put_device(pvt->chancounts_dev);
1221	}
1222	device_del(pvt->addrmatch_dev);
1223	put_device(pvt->addrmatch_dev);
1224}
1225
1226/****************************************************************************
1227	Device initialization routines: put/get, init/exit
1228 ****************************************************************************/
1229
1230/*
1231 *	i7core_put_all_devices	'put' all the devices that we have
1232 *				reserved via 'get'
1233 */
1234static void i7core_put_devices(struct i7core_dev *i7core_dev)
1235{
1236	int i;
1237
1238	edac_dbg(0, "\n");
1239	for (i = 0; i < i7core_dev->n_devs; i++) {
1240		struct pci_dev *pdev = i7core_dev->pdev[i];
1241		if (!pdev)
1242			continue;
1243		edac_dbg(0, "Removing dev %02x:%02x.%d\n",
1244			 pdev->bus->number,
1245			 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1246		pci_dev_put(pdev);
1247	}
1248}
1249
1250static void i7core_put_all_devices(void)
1251{
1252	struct i7core_dev *i7core_dev, *tmp;
1253
1254	list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1255		i7core_put_devices(i7core_dev);
1256		free_i7core_dev(i7core_dev);
1257	}
1258}
1259
1260static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1261{
1262	struct pci_dev *pdev = NULL;
1263	int i;
1264
1265	/*
1266	 * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1267	 * aren't announced by acpi. So, we need to use a legacy scan probing
1268	 * to detect them
1269	 */
1270	while (table && table->descr) {
1271		pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1272		if (unlikely(!pdev)) {
1273			for (i = 0; i < MAX_SOCKET_BUSES; i++)
1274				pcibios_scan_specific_bus(255-i);
1275		}
1276		pci_dev_put(pdev);
1277		table++;
1278	}
1279}
1280
1281static unsigned i7core_pci_lastbus(void)
1282{
1283	int last_bus = 0, bus;
1284	struct pci_bus *b = NULL;
1285
1286	while ((b = pci_find_next_bus(b)) != NULL) {
1287		bus = b->number;
1288		edac_dbg(0, "Found bus %d\n", bus);
1289		if (bus > last_bus)
1290			last_bus = bus;
1291	}
1292
1293	edac_dbg(0, "Last bus %d\n", last_bus);
1294
1295	return last_bus;
1296}
1297
1298/*
1299 *	i7core_get_all_devices	Find and perform 'get' operation on the MCH's
1300 *			device/functions we want to reference for this driver
1301 *
1302 *			Need to 'get' device 16 func 1 and func 2
1303 */
1304static int i7core_get_onedevice(struct pci_dev **prev,
1305				const struct pci_id_table *table,
1306				const unsigned devno,
1307				const unsigned last_bus)
1308{
1309	struct i7core_dev *i7core_dev;
1310	const struct pci_id_descr *dev_descr = &table->descr[devno];
1311
1312	struct pci_dev *pdev = NULL;
1313	u8 bus = 0;
1314	u8 socket = 0;
1315
1316	pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1317			      dev_descr->dev_id, *prev);
1318
1319	/*
1320	 * On Xeon 55xx, the Intel QuickPath Arch Generic Non-core regs
1321	 * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1322	 * to probe for the alternate address in case of failure
1323	 */
1324	if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev) {
1325		pci_dev_get(*prev);	/* pci_get_device will put it */
1326		pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1327				      PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1328	}
1329
1330	if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE &&
1331	    !pdev) {
1332		pci_dev_get(*prev);	/* pci_get_device will put it */
1333		pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1334				      PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1335				      *prev);
1336	}
1337
1338	if (!pdev) {
1339		if (*prev) {
1340			*prev = pdev;
1341			return 0;
1342		}
1343
1344		if (dev_descr->optional)
1345			return 0;
1346
1347		if (devno == 0)
1348			return -ENODEV;
1349
1350		i7core_printk(KERN_INFO,
1351			"Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1352			dev_descr->dev, dev_descr->func,
1353			PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1354
1355		/* End of list, leave */
1356		return -ENODEV;
1357	}
1358	bus = pdev->bus->number;
1359
1360	socket = last_bus - bus;
1361
1362	i7core_dev = get_i7core_dev(socket);
1363	if (!i7core_dev) {
1364		i7core_dev = alloc_i7core_dev(socket, table);
1365		if (!i7core_dev) {
1366			pci_dev_put(pdev);
1367			return -ENOMEM;
1368		}
1369	}
1370
1371	if (i7core_dev->pdev[devno]) {
1372		i7core_printk(KERN_ERR,
1373			"Duplicated device for "
1374			"dev %02x:%02x.%d PCI ID %04x:%04x\n",
1375			bus, dev_descr->dev, dev_descr->func,
1376			PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1377		pci_dev_put(pdev);
1378		return -ENODEV;
1379	}
1380
1381	i7core_dev->pdev[devno] = pdev;
1382
1383	/* Sanity check */
1384	if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1385			PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1386		i7core_printk(KERN_ERR,
1387			"Device PCI ID %04x:%04x "
1388			"has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1389			PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1390			bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1391			bus, dev_descr->dev, dev_descr->func);
1392		return -ENODEV;
1393	}
1394
1395	/* Be sure that the device is enabled */
1396	if (unlikely(pci_enable_device(pdev) < 0)) {
1397		i7core_printk(KERN_ERR,
1398			"Couldn't enable "
1399			"dev %02x:%02x.%d PCI ID %04x:%04x\n",
1400			bus, dev_descr->dev, dev_descr->func,
1401			PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1402		return -ENODEV;
1403	}
1404
1405	edac_dbg(0, "Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1406		 socket, bus, dev_descr->dev,
1407		 dev_descr->func,
1408		 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1409
1410	/*
1411	 * As stated on drivers/pci/search.c, the reference count for
1412	 * @from is always decremented if it is not %NULL. So, as we need
1413	 * to get all devices up to null, we need to do a get for the device
1414	 */
1415	pci_dev_get(pdev);
1416
1417	*prev = pdev;
1418
1419	return 0;
1420}
1421
1422static int i7core_get_all_devices(void)
1423{
1424	int i, rc, last_bus;
1425	struct pci_dev *pdev = NULL;
1426	const struct pci_id_table *table = pci_dev_table;
1427
1428	last_bus = i7core_pci_lastbus();
1429
1430	while (table && table->descr) {
1431		for (i = 0; i < table->n_devs; i++) {
1432			pdev = NULL;
1433			do {
1434				rc = i7core_get_onedevice(&pdev, table, i,
1435							  last_bus);
1436				if (rc < 0) {
1437					if (i == 0) {
1438						i = table->n_devs;
1439						break;
1440					}
1441					i7core_put_all_devices();
1442					return -ENODEV;
1443				}
1444			} while (pdev);
1445		}
1446		table++;
1447	}
1448
1449	return 0;
1450}
1451
1452static int mci_bind_devs(struct mem_ctl_info *mci,
1453			 struct i7core_dev *i7core_dev)
1454{
1455	struct i7core_pvt *pvt = mci->pvt_info;
1456	struct pci_dev *pdev;
1457	int i, func, slot;
1458	char *family;
1459
1460	pvt->is_registered = false;
1461	pvt->enable_scrub  = false;
1462	for (i = 0; i < i7core_dev->n_devs; i++) {
1463		pdev = i7core_dev->pdev[i];
1464		if (!pdev)
1465			continue;
1466
1467		func = PCI_FUNC(pdev->devfn);
1468		slot = PCI_SLOT(pdev->devfn);
1469		if (slot == 3) {
1470			if (unlikely(func > MAX_MCR_FUNC))
1471				goto error;
1472			pvt->pci_mcr[func] = pdev;
1473		} else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1474			if (unlikely(func > MAX_CHAN_FUNC))
1475				goto error;
1476			pvt->pci_ch[slot - 4][func] = pdev;
1477		} else if (!slot && !func) {
1478			pvt->pci_noncore = pdev;
1479
1480			/* Detect the processor family */
1481			switch (pdev->device) {
1482			case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1483				family = "Xeon 35xx/ i7core";
1484				pvt->enable_scrub = false;
1485				break;
1486			case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1487				family = "i7-800/i5-700";
1488				pvt->enable_scrub = false;
1489				break;
1490			case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1491				family = "Xeon 34xx";
1492				pvt->enable_scrub = false;
1493				break;
1494			case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1495				family = "Xeon 55xx";
1496				pvt->enable_scrub = true;
1497				break;
1498			case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1499				family = "Xeon 56xx / i7-900";
1500				pvt->enable_scrub = true;
1501				break;
1502			default:
1503				family = "unknown";
1504				pvt->enable_scrub = false;
1505			}
1506			edac_dbg(0, "Detected a processor type %s\n", family);
1507		} else
1508			goto error;
1509
1510		edac_dbg(0, "Associated fn %d.%d, dev = %p, socket %d\n",
1511			 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1512			 pdev, i7core_dev->socket);
1513
1514		if (PCI_SLOT(pdev->devfn) == 3 &&
1515			PCI_FUNC(pdev->devfn) == 2)
1516			pvt->is_registered = true;
1517	}
1518
1519	return 0;
1520
1521error:
1522	i7core_printk(KERN_ERR, "Device %d, function %d "
1523		      "is out of the expected range\n",
1524		      slot, func);
1525	return -EINVAL;
1526}
1527
1528/****************************************************************************
1529			Error check routines
1530 ****************************************************************************/
1531
1532static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1533					 const int chan,
1534					 const int new0,
1535					 const int new1,
1536					 const int new2)
1537{
1538	struct i7core_pvt *pvt = mci->pvt_info;
1539	int add0 = 0, add1 = 0, add2 = 0;
1540	/* Updates CE counters if it is not the first time here */
1541	if (pvt->ce_count_available) {
1542		/* Updates CE counters */
1543
1544		add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1545		add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1546		add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1547
1548		if (add2 < 0)
1549			add2 += 0x7fff;
1550		pvt->rdimm_ce_count[chan][2] += add2;
1551
1552		if (add1 < 0)
1553			add1 += 0x7fff;
1554		pvt->rdimm_ce_count[chan][1] += add1;
1555
1556		if (add0 < 0)
1557			add0 += 0x7fff;
1558		pvt->rdimm_ce_count[chan][0] += add0;
1559	} else
1560		pvt->ce_count_available = 1;
1561
1562	/* Store the new values */
1563	pvt->rdimm_last_ce_count[chan][2] = new2;
1564	pvt->rdimm_last_ce_count[chan][1] = new1;
1565	pvt->rdimm_last_ce_count[chan][0] = new0;
1566
1567	/*updated the edac core */
1568	if (add0 != 0)
1569		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add0,
1570				     0, 0, 0,
1571				     chan, 0, -1, "error", "");
1572	if (add1 != 0)
1573		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add1,
1574				     0, 0, 0,
1575				     chan, 1, -1, "error", "");
1576	if (add2 != 0)
1577		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, add2,
1578				     0, 0, 0,
1579				     chan, 2, -1, "error", "");
1580}
1581
1582static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1583{
1584	struct i7core_pvt *pvt = mci->pvt_info;
1585	u32 rcv[3][2];
1586	int i, new0, new1, new2;
1587
1588	/*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1589	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1590								&rcv[0][0]);
1591	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1592								&rcv[0][1]);
1593	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1594								&rcv[1][0]);
1595	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1596								&rcv[1][1]);
1597	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1598								&rcv[2][0]);
1599	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1600								&rcv[2][1]);
1601	for (i = 0 ; i < 3; i++) {
1602		edac_dbg(3, "MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1603			 (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1604		/*if the channel has 3 dimms*/
1605		if (pvt->channel[i].dimms > 2) {
1606			new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1607			new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1608			new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1609		} else {
1610			new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1611					DIMM_BOT_COR_ERR(rcv[i][0]);
1612			new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1613					DIMM_BOT_COR_ERR(rcv[i][1]);
1614			new2 = 0;
1615		}
1616
1617		i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1618	}
1619}
1620
1621/* This function is based on the device 3 function 4 registers as described on:
1622 * Intel Xeon Processor 5500 Series Datasheet Volume 2
1623 *	http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1624 * also available at:
1625 * 	http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1626 */
1627static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1628{
1629	struct i7core_pvt *pvt = mci->pvt_info;
1630	u32 rcv1, rcv0;
1631	int new0, new1, new2;
1632
1633	if (!pvt->pci_mcr[4]) {
1634		edac_dbg(0, "MCR registers not found\n");
1635		return;
1636	}
1637
1638	/* Corrected test errors */
1639	pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1640	pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1641
1642	/* Store the new values */
1643	new2 = DIMM2_COR_ERR(rcv1);
1644	new1 = DIMM1_COR_ERR(rcv0);
1645	new0 = DIMM0_COR_ERR(rcv0);
1646
1647	/* Updates CE counters if it is not the first time here */
1648	if (pvt->ce_count_available) {
1649		/* Updates CE counters */
1650		int add0, add1, add2;
1651
1652		add2 = new2 - pvt->udimm_last_ce_count[2];
1653		add1 = new1 - pvt->udimm_last_ce_count[1];
1654		add0 = new0 - pvt->udimm_last_ce_count[0];
1655
1656		if (add2 < 0)
1657			add2 += 0x7fff;
1658		pvt->udimm_ce_count[2] += add2;
1659
1660		if (add1 < 0)
1661			add1 += 0x7fff;
1662		pvt->udimm_ce_count[1] += add1;
1663
1664		if (add0 < 0)
1665			add0 += 0x7fff;
1666		pvt->udimm_ce_count[0] += add0;
1667
1668		if (add0 | add1 | add2)
1669			i7core_printk(KERN_ERR, "New Corrected error(s): "
1670				      "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1671				      add0, add1, add2);
1672	} else
1673		pvt->ce_count_available = 1;
1674
1675	/* Store the new values */
1676	pvt->udimm_last_ce_count[2] = new2;
1677	pvt->udimm_last_ce_count[1] = new1;
1678	pvt->udimm_last_ce_count[0] = new0;
1679}
1680
1681/*
1682 * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1683 * Architectures Software Developer’s Manual Volume 3B.
1684 * Nehalem are defined as family 0x06, model 0x1a
1685 *
1686 * The MCA registers used here are the following ones:
1687 *     struct mce field	MCA Register
1688 *     m->status	MSR_IA32_MC8_STATUS
1689 *     m->addr		MSR_IA32_MC8_ADDR
1690 *     m->misc		MSR_IA32_MC8_MISC
1691 * In the case of Nehalem, the error information is masked at .status and .misc
1692 * fields
1693 */
1694static void i7core_mce_output_error(struct mem_ctl_info *mci,
1695				    const struct mce *m)
1696{
1697	struct i7core_pvt *pvt = mci->pvt_info;
1698	char *optype, *err;
1699	enum hw_event_mc_err_type tp_event;
1700	unsigned long error = m->status & 0x1ff0000l;
1701	bool uncorrected_error = m->mcgstatus & 1ll << 61;
1702	bool ripv = m->mcgstatus & 1;
1703	u32 optypenum = (m->status >> 4) & 0x07;
1704	u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1705	u32 dimm = (m->misc >> 16) & 0x3;
1706	u32 channel = (m->misc >> 18) & 0x3;
1707	u32 syndrome = m->misc >> 32;
1708	u32 errnum = find_first_bit(&error, 32);
1709
1710	if (uncorrected_error) {
1711		core_err_cnt = 1;
1712		if (ripv)
1713			tp_event = HW_EVENT_ERR_UNCORRECTED;
1714		else
1715			tp_event = HW_EVENT_ERR_FATAL;
1716	} else {
1717		tp_event = HW_EVENT_ERR_CORRECTED;
1718	}
1719
1720	switch (optypenum) {
1721	case 0:
1722		optype = "generic undef request";
1723		break;
1724	case 1:
1725		optype = "read error";
1726		break;
1727	case 2:
1728		optype = "write error";
1729		break;
1730	case 3:
1731		optype = "addr/cmd error";
1732		break;
1733	case 4:
1734		optype = "scrubbing error";
1735		break;
1736	default:
1737		optype = "reserved";
1738		break;
1739	}
1740
1741	switch (errnum) {
1742	case 16:
1743		err = "read ECC error";
1744		break;
1745	case 17:
1746		err = "RAS ECC error";
1747		break;
1748	case 18:
1749		err = "write parity error";
1750		break;
1751	case 19:
1752		err = "redundancy loss";
1753		break;
1754	case 20:
1755		err = "reserved";
1756		break;
1757	case 21:
1758		err = "memory range error";
1759		break;
1760	case 22:
1761		err = "RTID out of range";
1762		break;
1763	case 23:
1764		err = "address parity error";
1765		break;
1766	case 24:
1767		err = "byte enable parity error";
1768		break;
1769	default:
1770		err = "unknown";
1771	}
1772
1773	/*
1774	 * Call the helper to output message
1775	 * FIXME: what to do if core_err_cnt > 1? Currently, it generates
1776	 * only one event
1777	 */
1778	if (uncorrected_error || !pvt->is_registered)
1779		edac_mc_handle_error(tp_event, mci, core_err_cnt,
1780				     m->addr >> PAGE_SHIFT,
1781				     m->addr & ~PAGE_MASK,
1782				     syndrome,
1783				     channel, dimm, -1,
1784				     err, optype);
1785}
1786
1787/*
1788 *	i7core_check_error	Retrieve and process errors reported by the
1789 *				hardware. Called by the Core module.
1790 */
1791static void i7core_check_error(struct mem_ctl_info *mci, struct mce *m)
1792{
1793	struct i7core_pvt *pvt = mci->pvt_info;
1794
1795	i7core_mce_output_error(mci, m);
1796
1797	/*
1798	 * Now, let's increment CE error counts
1799	 */
1800	if (!pvt->is_registered)
1801		i7core_udimm_check_mc_ecc_err(mci);
1802	else
1803		i7core_rdimm_check_mc_ecc_err(mci);
1804}
1805
1806/*
1807 * Check that logging is enabled and that this is the right type
1808 * of error for us to handle.
1809 */
1810static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1811				  void *data)
1812{
1813	struct mce *mce = (struct mce *)data;
1814	struct i7core_dev *i7_dev;
1815	struct mem_ctl_info *mci;
1816
1817	i7_dev = get_i7core_dev(mce->socketid);
1818	if (!i7_dev || (mce->kflags & MCE_HANDLED_CEC))
1819		return NOTIFY_DONE;
1820
1821	mci = i7_dev->mci;
1822
1823	/*
1824	 * Just let mcelog handle it if the error is
1825	 * outside the memory controller
1826	 */
1827	if (((mce->status & 0xffff) >> 7) != 1)
1828		return NOTIFY_DONE;
1829
1830	/* Bank 8 registers are the only ones that we know how to handle */
1831	if (mce->bank != 8)
1832		return NOTIFY_DONE;
1833
1834	i7core_check_error(mci, mce);
1835
1836	/* Advise mcelog that the errors were handled */
1837	mce->kflags |= MCE_HANDLED_EDAC;
1838	return NOTIFY_OK;
1839}
1840
1841static struct notifier_block i7_mce_dec = {
1842	.notifier_call	= i7core_mce_check_error,
1843	.priority	= MCE_PRIO_EDAC,
1844};
1845
1846struct memdev_dmi_entry {
1847	u8 type;
1848	u8 length;
1849	u16 handle;
1850	u16 phys_mem_array_handle;
1851	u16 mem_err_info_handle;
1852	u16 total_width;
1853	u16 data_width;
1854	u16 size;
1855	u8 form;
1856	u8 device_set;
1857	u8 device_locator;
1858	u8 bank_locator;
1859	u8 memory_type;
1860	u16 type_detail;
1861	u16 speed;
1862	u8 manufacturer;
1863	u8 serial_number;
1864	u8 asset_tag;
1865	u8 part_number;
1866	u8 attributes;
1867	u32 extended_size;
1868	u16 conf_mem_clk_speed;
1869} __attribute__((__packed__));
1870
1871
1872/*
1873 * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1874 * memory devices show the same speed, and if they don't then consider
1875 * all speeds to be invalid.
1876 */
1877static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1878{
1879	int *dclk_freq = _dclk_freq;
1880	u16 dmi_mem_clk_speed;
1881
1882	if (*dclk_freq == -1)
1883		return;
1884
1885	if (dh->type == DMI_ENTRY_MEM_DEVICE) {
1886		struct memdev_dmi_entry *memdev_dmi_entry =
1887			(struct memdev_dmi_entry *)dh;
1888		unsigned long conf_mem_clk_speed_offset =
1889			(unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
1890			(unsigned long)&memdev_dmi_entry->type;
1891		unsigned long speed_offset =
1892			(unsigned long)&memdev_dmi_entry->speed -
1893			(unsigned long)&memdev_dmi_entry->type;
1894
1895		/* Check that a DIMM is present */
1896		if (memdev_dmi_entry->size == 0)
1897			return;
1898
1899		/*
1900		 * Pick the configured speed if it's available, otherwise
1901		 * pick the DIMM speed, or we don't have a speed.
1902		 */
1903		if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
1904			dmi_mem_clk_speed =
1905				memdev_dmi_entry->conf_mem_clk_speed;
1906		} else if (memdev_dmi_entry->length > speed_offset) {
1907			dmi_mem_clk_speed = memdev_dmi_entry->speed;
1908		} else {
1909			*dclk_freq = -1;
1910			return;
1911		}
1912
1913		if (*dclk_freq == 0) {
1914			/* First pass, speed was 0 */
1915			if (dmi_mem_clk_speed > 0) {
1916				/* Set speed if a valid speed is read */
1917				*dclk_freq = dmi_mem_clk_speed;
1918			} else {
1919				/* Otherwise we don't have a valid speed */
1920				*dclk_freq = -1;
1921			}
1922		} else if (*dclk_freq > 0 &&
1923			   *dclk_freq != dmi_mem_clk_speed) {
1924			/*
1925			 * If we have a speed, check that all DIMMS are the same
1926			 * speed, otherwise set the speed as invalid.
1927			 */
1928			*dclk_freq = -1;
1929		}
1930	}
1931}
1932
1933/*
1934 * The default DCLK frequency is used as a fallback if we
1935 * fail to find anything reliable in the DMI. The value
1936 * is taken straight from the datasheet.
1937 */
1938#define DEFAULT_DCLK_FREQ 800
1939
1940static int get_dclk_freq(void)
1941{
1942	int dclk_freq = 0;
1943
1944	dmi_walk(decode_dclk, (void *)&dclk_freq);
1945
1946	if (dclk_freq < 1)
1947		return DEFAULT_DCLK_FREQ;
1948
1949	return dclk_freq;
1950}
1951
1952/*
1953 * set_sdram_scrub_rate		This routine sets byte/sec bandwidth scrub rate
1954 *				to hardware according to SCRUBINTERVAL formula
1955 *				found in datasheet.
1956 */
1957static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
1958{
1959	struct i7core_pvt *pvt = mci->pvt_info;
1960	struct pci_dev *pdev;
1961	u32 dw_scrub;
1962	u32 dw_ssr;
1963
1964	/* Get data from the MC register, function 2 */
1965	pdev = pvt->pci_mcr[2];
1966	if (!pdev)
1967		return -ENODEV;
1968
1969	pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
1970
1971	if (new_bw == 0) {
1972		/* Prepare to disable petrol scrub */
1973		dw_scrub &= ~STARTSCRUB;
1974		/* Stop the patrol scrub engine */
1975		write_and_test(pdev, MC_SCRUB_CONTROL,
1976			       dw_scrub & ~SCRUBINTERVAL_MASK);
1977
1978		/* Get current status of scrub rate and set bit to disable */
1979		pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
1980		dw_ssr &= ~SSR_MODE_MASK;
1981		dw_ssr |= SSR_MODE_DISABLE;
1982	} else {
1983		const int cache_line_size = 64;
1984		const u32 freq_dclk_mhz = pvt->dclk_freq;
1985		unsigned long long scrub_interval;
1986		/*
1987		 * Translate the desired scrub rate to a register value and
1988		 * program the corresponding register value.
1989		 */
1990		scrub_interval = (unsigned long long)freq_dclk_mhz *
1991			cache_line_size * 1000000;
1992		do_div(scrub_interval, new_bw);
1993
1994		if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
1995			return -EINVAL;
1996
1997		dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
1998
1999		/* Start the patrol scrub engine */
2000		pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
2001				       STARTSCRUB | dw_scrub);
2002
2003		/* Get current status of scrub rate and set bit to enable */
2004		pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2005		dw_ssr &= ~SSR_MODE_MASK;
2006		dw_ssr |= SSR_MODE_ENABLE;
2007	}
2008	/* Disable or enable scrubbing */
2009	pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2010
2011	return new_bw;
2012}
2013
2014/*
2015 * get_sdram_scrub_rate		This routine convert current scrub rate value
2016 *				into byte/sec bandwidth according to
2017 *				SCRUBINTERVAL formula found in datasheet.
2018 */
2019static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2020{
2021	struct i7core_pvt *pvt = mci->pvt_info;
2022	struct pci_dev *pdev;
2023	const u32 cache_line_size = 64;
2024	const u32 freq_dclk_mhz = pvt->dclk_freq;
2025	unsigned long long scrub_rate;
2026	u32 scrubval;
2027
2028	/* Get data from the MC register, function 2 */
2029	pdev = pvt->pci_mcr[2];
2030	if (!pdev)
2031		return -ENODEV;
2032
2033	/* Get current scrub control data */
2034	pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2035
2036	/* Mask highest 8-bits to 0 */
2037	scrubval &=  SCRUBINTERVAL_MASK;
2038	if (!scrubval)
2039		return 0;
2040
2041	/* Calculate scrub rate value into byte/sec bandwidth */
2042	scrub_rate =  (unsigned long long)freq_dclk_mhz *
2043		1000000 * cache_line_size;
2044	do_div(scrub_rate, scrubval);
2045	return (int)scrub_rate;
2046}
2047
2048static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2049{
2050	struct i7core_pvt *pvt = mci->pvt_info;
2051	u32 pci_lock;
2052
2053	/* Unlock writes to pci registers */
2054	pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2055	pci_lock &= ~0x3;
2056	pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2057			       pci_lock | MC_CFG_UNLOCK);
2058
2059	mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2060	mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2061}
2062
2063static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2064{
2065	struct i7core_pvt *pvt = mci->pvt_info;
2066	u32 pci_lock;
2067
2068	/* Lock writes to pci registers */
2069	pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2070	pci_lock &= ~0x3;
2071	pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2072			       pci_lock | MC_CFG_LOCK);
2073}
2074
2075static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2076{
2077	pvt->i7core_pci = edac_pci_create_generic_ctl(
2078						&pvt->i7core_dev->pdev[0]->dev,
2079						EDAC_MOD_STR);
2080	if (unlikely(!pvt->i7core_pci))
2081		i7core_printk(KERN_WARNING,
2082			      "Unable to setup PCI error report via EDAC\n");
2083}
2084
2085static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2086{
2087	if (likely(pvt->i7core_pci))
2088		edac_pci_release_generic_ctl(pvt->i7core_pci);
2089	else
2090		i7core_printk(KERN_ERR,
2091				"Couldn't find mem_ctl_info for socket %d\n",
2092				pvt->i7core_dev->socket);
2093	pvt->i7core_pci = NULL;
2094}
2095
2096static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2097{
2098	struct mem_ctl_info *mci = i7core_dev->mci;
2099	struct i7core_pvt *pvt;
2100
2101	if (unlikely(!mci || !mci->pvt_info)) {
2102		edac_dbg(0, "MC: dev = %p\n", &i7core_dev->pdev[0]->dev);
2103
2104		i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2105		return;
2106	}
2107
2108	pvt = mci->pvt_info;
2109
2110	edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev);
2111
2112	/* Disable scrubrate setting */
2113	if (pvt->enable_scrub)
2114		disable_sdram_scrub_setting(mci);
2115
2116	/* Disable EDAC polling */
2117	i7core_pci_ctl_release(pvt);
2118
2119	/* Remove MC sysfs nodes */
2120	i7core_delete_sysfs_devices(mci);
2121	edac_mc_del_mc(mci->pdev);
2122
2123	edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
2124	kfree(mci->ctl_name);
2125	edac_mc_free(mci);
2126	i7core_dev->mci = NULL;
2127}
2128
2129static int i7core_register_mci(struct i7core_dev *i7core_dev)
2130{
2131	struct mem_ctl_info *mci;
2132	struct i7core_pvt *pvt;
2133	int rc;
2134	struct edac_mc_layer layers[2];
2135
2136	/* allocate a new MC control structure */
2137
2138	layers[0].type = EDAC_MC_LAYER_CHANNEL;
2139	layers[0].size = NUM_CHANS;
2140	layers[0].is_virt_csrow = false;
2141	layers[1].type = EDAC_MC_LAYER_SLOT;
2142	layers[1].size = MAX_DIMMS;
2143	layers[1].is_virt_csrow = true;
2144	mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers,
2145			    sizeof(*pvt));
2146	if (unlikely(!mci))
2147		return -ENOMEM;
2148
2149	edac_dbg(0, "MC: mci = %p, dev = %p\n", mci, &i7core_dev->pdev[0]->dev);
2150
2151	pvt = mci->pvt_info;
2152	memset(pvt, 0, sizeof(*pvt));
2153
2154	/* Associates i7core_dev and mci for future usage */
2155	pvt->i7core_dev = i7core_dev;
2156	i7core_dev->mci = mci;
2157
2158	/*
2159	 * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2160	 * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2161	 * memory channels
2162	 */
2163	mci->mtype_cap = MEM_FLAG_DDR3;
2164	mci->edac_ctl_cap = EDAC_FLAG_NONE;
2165	mci->edac_cap = EDAC_FLAG_NONE;
2166	mci->mod_name = "i7core_edac.c";
2167
2168	mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d", i7core_dev->socket);
2169	if (!mci->ctl_name) {
2170		rc = -ENOMEM;
2171		goto fail1;
2172	}
2173
2174	mci->dev_name = pci_name(i7core_dev->pdev[0]);
2175	mci->ctl_page_to_phys = NULL;
2176
2177	/* Store pci devices at mci for faster access */
2178	rc = mci_bind_devs(mci, i7core_dev);
2179	if (unlikely(rc < 0))
2180		goto fail0;
2181
2182
2183	/* Get dimm basic config */
2184	get_dimm_config(mci);
2185	/* record ptr to the generic device */
2186	mci->pdev = &i7core_dev->pdev[0]->dev;
2187
2188	/* Enable scrubrate setting */
2189	if (pvt->enable_scrub)
2190		enable_sdram_scrub_setting(mci);
2191
2192	/* add this new MC control structure to EDAC's list of MCs */
2193	if (unlikely(edac_mc_add_mc_with_groups(mci, i7core_dev_groups))) {
2194		edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
2195		/* FIXME: perhaps some code should go here that disables error
2196		 * reporting if we just enabled it
2197		 */
2198
2199		rc = -EINVAL;
2200		goto fail0;
2201	}
2202	if (i7core_create_sysfs_devices(mci)) {
2203		edac_dbg(0, "MC: failed to create sysfs nodes\n");
2204		edac_mc_del_mc(mci->pdev);
2205		rc = -EINVAL;
2206		goto fail0;
2207	}
2208
2209	/* Default error mask is any memory */
2210	pvt->inject.channel = 0;
2211	pvt->inject.dimm = -1;
2212	pvt->inject.rank = -1;
2213	pvt->inject.bank = -1;
2214	pvt->inject.page = -1;
2215	pvt->inject.col = -1;
2216
2217	/* allocating generic PCI control info */
2218	i7core_pci_ctl_create(pvt);
2219
2220	/* DCLK for scrub rate setting */
2221	pvt->dclk_freq = get_dclk_freq();
2222
2223	return 0;
2224
2225fail0:
2226	kfree(mci->ctl_name);
2227
2228fail1:
2229	edac_mc_free(mci);
2230	i7core_dev->mci = NULL;
2231	return rc;
2232}
2233
2234/*
2235 *	i7core_probe	Probe for ONE instance of device to see if it is
2236 *			present.
2237 *	return:
2238 *		0 for FOUND a device
2239 *		< 0 for error code
2240 */
2241
2242static int i7core_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2243{
2244	int rc, count = 0;
2245	struct i7core_dev *i7core_dev;
2246
2247	/* get the pci devices we want to reserve for our use */
2248	mutex_lock(&i7core_edac_lock);
2249
2250	/*
2251	 * All memory controllers are allocated at the first pass.
2252	 */
2253	if (unlikely(probed >= 1)) {
2254		mutex_unlock(&i7core_edac_lock);
2255		return -ENODEV;
2256	}
2257	probed++;
2258
2259	rc = i7core_get_all_devices();
2260	if (unlikely(rc < 0))
2261		goto fail0;
2262
2263	list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2264		count++;
2265		rc = i7core_register_mci(i7core_dev);
2266		if (unlikely(rc < 0))
2267			goto fail1;
2268	}
2269
2270	/*
2271	 * Nehalem-EX uses a different memory controller. However, as the
2272	 * memory controller is not visible on some Nehalem/Nehalem-EP, we
2273	 * need to indirectly probe via a X58 PCI device. The same devices
2274	 * are found on (some) Nehalem-EX. So, on those machines, the
2275	 * probe routine needs to return -ENODEV, as the actual Memory
2276	 * Controller registers won't be detected.
2277	 */
2278	if (!count) {
2279		rc = -ENODEV;
2280		goto fail1;
2281	}
2282
2283	i7core_printk(KERN_INFO,
2284		      "Driver loaded, %d memory controller(s) found.\n",
2285		      count);
2286
2287	mutex_unlock(&i7core_edac_lock);
2288	return 0;
2289
2290fail1:
2291	list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2292		i7core_unregister_mci(i7core_dev);
2293
2294	i7core_put_all_devices();
2295fail0:
2296	mutex_unlock(&i7core_edac_lock);
2297	return rc;
2298}
2299
2300/*
2301 *	i7core_remove	destructor for one instance of device
2302 *
2303 */
2304static void i7core_remove(struct pci_dev *pdev)
2305{
2306	struct i7core_dev *i7core_dev;
2307
2308	edac_dbg(0, "\n");
2309
2310	/*
2311	 * we have a trouble here: pdev value for removal will be wrong, since
2312	 * it will point to the X58 register used to detect that the machine
2313	 * is a Nehalem or upper design. However, due to the way several PCI
2314	 * devices are grouped together to provide MC functionality, we need
2315	 * to use a different method for releasing the devices
2316	 */
2317
2318	mutex_lock(&i7core_edac_lock);
2319
2320	if (unlikely(!probed)) {
2321		mutex_unlock(&i7core_edac_lock);
2322		return;
2323	}
2324
2325	list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2326		i7core_unregister_mci(i7core_dev);
2327
2328	/* Release PCI resources */
2329	i7core_put_all_devices();
2330
2331	probed--;
2332
2333	mutex_unlock(&i7core_edac_lock);
2334}
2335
2336MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2337
2338/*
2339 *	i7core_driver	pci_driver structure for this module
2340 *
2341 */
2342static struct pci_driver i7core_driver = {
2343	.name     = "i7core_edac",
2344	.probe    = i7core_probe,
2345	.remove   = i7core_remove,
2346	.id_table = i7core_pci_tbl,
2347};
2348
2349/*
2350 *	i7core_init		Module entry function
2351 *			Try to initialize this module for its devices
2352 */
2353static int __init i7core_init(void)
2354{
2355	int pci_rc;
2356
2357	edac_dbg(2, "\n");
2358
2359	/* Ensure that the OPSTATE is set correctly for POLL or NMI */
2360	opstate_init();
2361
2362	if (use_pci_fixup)
2363		i7core_xeon_pci_fixup(pci_dev_table);
2364
2365	pci_rc = pci_register_driver(&i7core_driver);
2366
2367	if (pci_rc >= 0) {
2368		mce_register_decode_chain(&i7_mce_dec);
2369		return 0;
2370	}
2371
2372	i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2373		      pci_rc);
2374
2375	return pci_rc;
2376}
2377
2378/*
2379 *	i7core_exit()	Module exit function
2380 *			Unregister the driver
2381 */
2382static void __exit i7core_exit(void)
2383{
2384	edac_dbg(2, "\n");
2385	pci_unregister_driver(&i7core_driver);
2386	mce_unregister_decode_chain(&i7_mce_dec);
2387}
2388
2389module_init(i7core_init);
2390module_exit(i7core_exit);
2391
2392MODULE_LICENSE("GPL");
2393MODULE_AUTHOR("Mauro Carvalho Chehab");
2394MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
2395MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2396		   I7CORE_REVISION);
2397
2398module_param(edac_op_state, int, 0444);
2399MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");