1/*
   2 * Copyright IBM Corporation 2001, 2005, 2006
   3 * Copyright Dave Engebretsen & Todd Inglett 2001
   4 * Copyright Linas Vepstas 2005, 2006
   5 * Copyright 2001-2012 IBM Corporation.
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 2 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * This program is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  20 *
  21 * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com>
  22 */
  23
  24#include <linux/delay.h>
 
  25#include <linux/sched.h>
  26#include <linux/init.h>
  27#include <linux/list.h>
  28#include <linux/pci.h>
  29#include <linux/iommu.h>
  30#include <linux/proc_fs.h>
  31#include <linux/rbtree.h>
  32#include <linux/reboot.h>
  33#include <linux/seq_file.h>
  34#include <linux/spinlock.h>
  35#include <linux/export.h>
  36#include <linux/of.h>
  37
  38#include <linux/atomic.h>
  39#include <asm/debugfs.h>
  40#include <asm/eeh.h>
  41#include <asm/eeh_event.h>
  42#include <asm/io.h>
  43#include <asm/iommu.h>
  44#include <asm/machdep.h>
  45#include <asm/ppc-pci.h>
  46#include <asm/rtas.h>
  47#include <asm/pte-walk.h>
  48
  49
  50/** Overview:
  51 *  EEH, or "Enhanced Error Handling" is a PCI bridge technology for
  52 *  dealing with PCI bus errors that can't be dealt with within the
  53 *  usual PCI framework, except by check-stopping the CPU.  Systems
  54 *  that are designed for high-availability/reliability cannot afford
  55 *  to crash due to a "mere" PCI error, thus the need for EEH.
  56 *  An EEH-capable bridge operates by converting a detected error
  57 *  into a "slot freeze", taking the PCI adapter off-line, making
  58 *  the slot behave, from the OS'es point of view, as if the slot
  59 *  were "empty": all reads return 0xff's and all writes are silently
  60 *  ignored.  EEH slot isolation events can be triggered by parity
  61 *  errors on the address or data busses (e.g. during posted writes),
  62 *  which in turn might be caused by low voltage on the bus, dust,
  63 *  vibration, humidity, radioactivity or plain-old failed hardware.
  64 *
  65 *  Note, however, that one of the leading causes of EEH slot
  66 *  freeze events are buggy device drivers, buggy device microcode,
  67 *  or buggy device hardware.  This is because any attempt by the
  68 *  device to bus-master data to a memory address that is not
  69 *  assigned to the device will trigger a slot freeze.   (The idea
  70 *  is to prevent devices-gone-wild from corrupting system memory).
  71 *  Buggy hardware/drivers will have a miserable time co-existing
  72 *  with EEH.
  73 *
  74 *  Ideally, a PCI device driver, when suspecting that an isolation
  75 *  event has occurred (e.g. by reading 0xff's), will then ask EEH
  76 *  whether this is the case, and then take appropriate steps to
  77 *  reset the PCI slot, the PCI device, and then resume operations.
  78 *  However, until that day,  the checking is done here, with the
  79 *  eeh_check_failure() routine embedded in the MMIO macros.  If
  80 *  the slot is found to be isolated, an "EEH Event" is synthesized
  81 *  and sent out for processing.
  82 */
  83
  84/* If a device driver keeps reading an MMIO register in an interrupt
  85 * handler after a slot isolation event, it might be broken.
  86 * This sets the threshold for how many read attempts we allow
  87 * before printing an error message.
  88 */
  89#define EEH_MAX_FAILS	2100000
  90
  91/* Time to wait for a PCI slot to report status, in milliseconds */
  92#define PCI_BUS_RESET_WAIT_MSEC (5*60*1000)
  93
  94/*
  95 * EEH probe mode support, which is part of the flags,
  96 * is to support multiple platforms for EEH. Some platforms
  97 * like pSeries do PCI emunation based on device tree.
  98 * However, other platforms like powernv probe PCI devices
  99 * from hardware. The flag is used to distinguish that.
 100 * In addition, struct eeh_ops::probe would be invoked for
 101 * particular OF node or PCI device so that the corresponding
 102 * PE would be created there.
 103 */
 104int eeh_subsystem_flags;
 105EXPORT_SYMBOL(eeh_subsystem_flags);
 106
 107/*
 108 * EEH allowed maximal frozen times. If one particular PE's
 109 * frozen count in last hour exceeds this limit, the PE will
 110 * be forced to be offline permanently.
 111 */
 112int eeh_max_freezes = 5;
 113
 114/* Platform dependent EEH operations */
 115struct eeh_ops *eeh_ops = NULL;
 116
 117/* Lock to avoid races due to multiple reports of an error */
 118DEFINE_RAW_SPINLOCK(confirm_error_lock);
 119EXPORT_SYMBOL_GPL(confirm_error_lock);
 120
 121/* Lock to protect passed flags */
 122static DEFINE_MUTEX(eeh_dev_mutex);
 123
 124/* Buffer for reporting pci register dumps. Its here in BSS, and
 125 * not dynamically alloced, so that it ends up in RMO where RTAS
 126 * can access it.
 127 */
 128#define EEH_PCI_REGS_LOG_LEN 8192
 129static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN];
 130
 131/*
 132 * The struct is used to maintain the EEH global statistic
 133 * information. Besides, the EEH global statistics will be
 134 * exported to user space through procfs
 135 */
 136struct eeh_stats {
 137	u64 no_device;		/* PCI device not found		*/
 138	u64 no_dn;		/* OF node not found		*/
 139	u64 no_cfg_addr;	/* Config address not found	*/
 140	u64 ignored_check;	/* EEH check skipped		*/
 141	u64 total_mmio_ffs;	/* Total EEH checks		*/
 142	u64 false_positives;	/* Unnecessary EEH checks	*/
 143	u64 slot_resets;	/* PE reset			*/
 144};
 145
 146static struct eeh_stats eeh_stats;
 147
 148static int __init eeh_setup(char *str)
 149{
 150	if (!strcmp(str, "off"))
 151		eeh_add_flag(EEH_FORCE_DISABLED);
 152	else if (!strcmp(str, "early_log"))
 153		eeh_add_flag(EEH_EARLY_DUMP_LOG);
 154
 155	return 1;
 156}
 157__setup("eeh=", eeh_setup);
 158
 159/*
 160 * This routine captures assorted PCI configuration space data
 161 * for the indicated PCI device, and puts them into a buffer
 162 * for RTAS error logging.
 163 */
 164static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len)
 165{
 166	struct pci_dn *pdn = eeh_dev_to_pdn(edev);
 167	u32 cfg;
 168	int cap, i;
 169	int n = 0, l = 0;
 170	char buffer[128];
 171
 172	n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n",
 173		       pdn->phb->global_number, pdn->busno,
 174		       PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
 175	pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n",
 176		pdn->phb->global_number, pdn->busno,
 177		PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
 178
 179	eeh_ops->read_config(pdn, PCI_VENDOR_ID, 4, &cfg);
 180	n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg);
 181	pr_warn("EEH: PCI device/vendor: %08x\n", cfg);
 182
 183	eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cfg);
 184	n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg);
 185	pr_warn("EEH: PCI cmd/status register: %08x\n", cfg);
 186
 187	/* Gather bridge-specific registers */
 188	if (edev->mode & EEH_DEV_BRIDGE) {
 189		eeh_ops->read_config(pdn, PCI_SEC_STATUS, 2, &cfg);
 190		n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg);
 191		pr_warn("EEH: Bridge secondary status: %04x\n", cfg);
 192
 193		eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &cfg);
 194		n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg);
 195		pr_warn("EEH: Bridge control: %04x\n", cfg);
 196	}
 197
 198	/* Dump out the PCI-X command and status regs */
 199	cap = edev->pcix_cap;
 200	if (cap) {
 201		eeh_ops->read_config(pdn, cap, 4, &cfg);
 202		n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg);
 203		pr_warn("EEH: PCI-X cmd: %08x\n", cfg);
 204
 205		eeh_ops->read_config(pdn, cap+4, 4, &cfg);
 206		n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg);
 207		pr_warn("EEH: PCI-X status: %08x\n", cfg);
 208	}
 209
 210	/* If PCI-E capable, dump PCI-E cap 10 */
 211	cap = edev->pcie_cap;
 212	if (cap) {
 213		n += scnprintf(buf+n, len-n, "pci-e cap10:\n");
 214		pr_warn("EEH: PCI-E capabilities and status follow:\n");
 215
 216		for (i=0; i<=8; i++) {
 217			eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
 218			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
 219
 220			if ((i % 4) == 0) {
 221				if (i != 0)
 222					pr_warn("%s\n", buffer);
 223
 224				l = scnprintf(buffer, sizeof(buffer),
 225					      "EEH: PCI-E %02x: %08x ",
 226					      4*i, cfg);
 227			} else {
 228				l += scnprintf(buffer+l, sizeof(buffer)-l,
 229					       "%08x ", cfg);
 230			}
 231
 232		}
 233
 234		pr_warn("%s\n", buffer);
 235	}
 236
 237	/* If AER capable, dump it */
 238	cap = edev->aer_cap;
 239	if (cap) {
 240		n += scnprintf(buf+n, len-n, "pci-e AER:\n");
 241		pr_warn("EEH: PCI-E AER capability register set follows:\n");
 242
 243		for (i=0; i<=13; i++) {
 244			eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
 245			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
 246
 247			if ((i % 4) == 0) {
 248				if (i != 0)
 249					pr_warn("%s\n", buffer);
 250
 251				l = scnprintf(buffer, sizeof(buffer),
 252					      "EEH: PCI-E AER %02x: %08x ",
 253					      4*i, cfg);
 254			} else {
 255				l += scnprintf(buffer+l, sizeof(buffer)-l,
 256					       "%08x ", cfg);
 257			}
 258		}
 259
 260		pr_warn("%s\n", buffer);
 261	}
 262
 263	return n;
 264}
 265
 266static void *eeh_dump_pe_log(void *data, void *flag)
 267{
 268	struct eeh_pe *pe = data;
 269	struct eeh_dev *edev, *tmp;
 270	size_t *plen = flag;
 271
 272	eeh_pe_for_each_dev(pe, edev, tmp)
 273		*plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen,
 274					  EEH_PCI_REGS_LOG_LEN - *plen);
 275
 276	return NULL;
 277}
 278
 279/**
 280 * eeh_slot_error_detail - Generate combined log including driver log and error log
 281 * @pe: EEH PE
 282 * @severity: temporary or permanent error log
 283 *
 284 * This routine should be called to generate the combined log, which
 285 * is comprised of driver log and error log. The driver log is figured
 286 * out from the config space of the corresponding PCI device, while
 287 * the error log is fetched through platform dependent function call.
 288 */
 289void eeh_slot_error_detail(struct eeh_pe *pe, int severity)
 290{
 291	size_t loglen = 0;
 292
 293	/*
 294	 * When the PHB is fenced or dead, it's pointless to collect
 295	 * the data from PCI config space because it should return
 296	 * 0xFF's. For ER, we still retrieve the data from the PCI
 297	 * config space.
 298	 *
 299	 * For pHyp, we have to enable IO for log retrieval. Otherwise,
 300	 * 0xFF's is always returned from PCI config space.
 301	 *
 302	 * When the @severity is EEH_LOG_PERM, the PE is going to be
 303	 * removed. Prior to that, the drivers for devices included in
 304	 * the PE will be closed. The drivers rely on working IO path
 305	 * to bring the devices to quiet state. Otherwise, PCI traffic
 306	 * from those devices after they are removed is like to cause
 307	 * another unexpected EEH error.
 308	 */
 309	if (!(pe->type & EEH_PE_PHB)) {
 310		if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) ||
 311		    severity == EEH_LOG_PERM)
 312			eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
 313
 314		/*
 315		 * The config space of some PCI devices can't be accessed
 316		 * when their PEs are in frozen state. Otherwise, fenced
 317		 * PHB might be seen. Those PEs are identified with flag
 318		 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED
 319		 * is set automatically when the PE is put to EEH_PE_ISOLATED.
 320		 *
 321		 * Restoring BARs possibly triggers PCI config access in
 322		 * (OPAL) firmware and then causes fenced PHB. If the
 323		 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's
 324		 * pointless to restore BARs and dump config space.
 325		 */
 326		eeh_ops->configure_bridge(pe);
 327		if (!(pe->state & EEH_PE_CFG_BLOCKED)) {
 328			eeh_pe_restore_bars(pe);
 329
 330			pci_regs_buf[0] = 0;
 331			eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen);
 332		}
 333	}
 334
 335	eeh_ops->get_log(pe, severity, pci_regs_buf, loglen);
 336}
 337
 338/**
 339 * eeh_token_to_phys - Convert EEH address token to phys address
 340 * @token: I/O token, should be address in the form 0xA....
 341 *
 342 * This routine should be called to convert virtual I/O address
 343 * to physical one.
 344 */
 345static inline unsigned long eeh_token_to_phys(unsigned long token)
 346{
 347	pte_t *ptep;
 348	unsigned long pa;
 349	int hugepage_shift;
 350
 351	/*
 352	 * We won't find hugepages here(this is iomem). Hence we are not
 353	 * worried about _PAGE_SPLITTING/collapse. Also we will not hit
 354	 * page table free, because of init_mm.
 355	 */
 356	ptep = find_init_mm_pte(token, &hugepage_shift);
 
 357	if (!ptep)
 358		return token;
 359	WARN_ON(hugepage_shift);
 360	pa = pte_pfn(*ptep) << PAGE_SHIFT;
 361
 362	return pa | (token & (PAGE_SIZE-1));
 363}
 364
 365/*
 366 * On PowerNV platform, we might already have fenced PHB there.
 367 * For that case, it's meaningless to recover frozen PE. Intead,
 368 * We have to handle fenced PHB firstly.
 369 */
 370static int eeh_phb_check_failure(struct eeh_pe *pe)
 371{
 372	struct eeh_pe *phb_pe;
 373	unsigned long flags;
 374	int ret;
 375
 376	if (!eeh_has_flag(EEH_PROBE_MODE_DEV))
 377		return -EPERM;
 378
 379	/* Find the PHB PE */
 380	phb_pe = eeh_phb_pe_get(pe->phb);
 381	if (!phb_pe) {
 382		pr_warn("%s Can't find PE for PHB#%x\n",
 383			__func__, pe->phb->global_number);
 384		return -EEXIST;
 385	}
 386
 387	/* If the PHB has been in problematic state */
 388	eeh_serialize_lock(&flags);
 389	if (phb_pe->state & EEH_PE_ISOLATED) {
 390		ret = 0;
 391		goto out;
 392	}
 393
 394	/* Check PHB state */
 395	ret = eeh_ops->get_state(phb_pe, NULL);
 396	if ((ret < 0) ||
 397	    (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) {
 
 
 398		ret = 0;
 399		goto out;
 400	}
 401
 402	/* Isolate the PHB and send event */
 403	eeh_pe_state_mark(phb_pe, EEH_PE_ISOLATED);
 404	eeh_serialize_unlock(flags);
 405
 406	pr_err("EEH: PHB#%x failure detected, location: %s\n",
 407		phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe));
 408	dump_stack();
 409	eeh_send_failure_event(phb_pe);
 410
 411	return 1;
 412out:
 413	eeh_serialize_unlock(flags);
 414	return ret;
 415}
 416
 417/**
 418 * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze
 419 * @edev: eeh device
 420 *
 421 * Check for an EEH failure for the given device node.  Call this
 422 * routine if the result of a read was all 0xff's and you want to
 423 * find out if this is due to an EEH slot freeze.  This routine
 424 * will query firmware for the EEH status.
 425 *
 426 * Returns 0 if there has not been an EEH error; otherwise returns
 427 * a non-zero value and queues up a slot isolation event notification.
 428 *
 429 * It is safe to call this routine in an interrupt context.
 430 */
 431int eeh_dev_check_failure(struct eeh_dev *edev)
 432{
 433	int ret;
 
 434	unsigned long flags;
 435	struct device_node *dn;
 436	struct pci_dev *dev;
 437	struct eeh_pe *pe, *parent_pe, *phb_pe;
 438	int rc = 0;
 439	const char *location = NULL;
 440
 441	eeh_stats.total_mmio_ffs++;
 442
 443	if (!eeh_enabled())
 444		return 0;
 445
 446	if (!edev) {
 447		eeh_stats.no_dn++;
 448		return 0;
 449	}
 450	dev = eeh_dev_to_pci_dev(edev);
 451	pe = eeh_dev_to_pe(edev);
 452
 453	/* Access to IO BARs might get this far and still not want checking. */
 454	if (!pe) {
 455		eeh_stats.ignored_check++;
 456		pr_debug("EEH: Ignored check for %s\n",
 457			eeh_pci_name(dev));
 458		return 0;
 459	}
 460
 461	if (!pe->addr && !pe->config_addr) {
 462		eeh_stats.no_cfg_addr++;
 463		return 0;
 464	}
 465
 466	/*
 467	 * On PowerNV platform, we might already have fenced PHB
 468	 * there and we need take care of that firstly.
 469	 */
 470	ret = eeh_phb_check_failure(pe);
 471	if (ret > 0)
 472		return ret;
 473
 474	/*
 475	 * If the PE isn't owned by us, we shouldn't check the
 476	 * state. Instead, let the owner handle it if the PE has
 477	 * been frozen.
 478	 */
 479	if (eeh_pe_passed(pe))
 480		return 0;
 481
 482	/* If we already have a pending isolation event for this
 483	 * slot, we know it's bad already, we don't need to check.
 484	 * Do this checking under a lock; as multiple PCI devices
 485	 * in one slot might report errors simultaneously, and we
 486	 * only want one error recovery routine running.
 487	 */
 488	eeh_serialize_lock(&flags);
 489	rc = 1;
 490	if (pe->state & EEH_PE_ISOLATED) {
 491		pe->check_count++;
 492		if (pe->check_count % EEH_MAX_FAILS == 0) {
 493			dn = pci_device_to_OF_node(dev);
 494			if (dn)
 495				location = of_get_property(dn, "ibm,loc-code",
 496						NULL);
 497			printk(KERN_ERR "EEH: %d reads ignored for recovering device at "
 498				"location=%s driver=%s pci addr=%s\n",
 499				pe->check_count,
 500				location ? location : "unknown",
 501				eeh_driver_name(dev), eeh_pci_name(dev));
 502			printk(KERN_ERR "EEH: Might be infinite loop in %s driver\n",
 503				eeh_driver_name(dev));
 504			dump_stack();
 505		}
 506		goto dn_unlock;
 507	}
 508
 509	/*
 510	 * Now test for an EEH failure.  This is VERY expensive.
 511	 * Note that the eeh_config_addr may be a parent device
 512	 * in the case of a device behind a bridge, or it may be
 513	 * function zero of a multi-function device.
 514	 * In any case they must share a common PHB.
 515	 */
 516	ret = eeh_ops->get_state(pe, NULL);
 517
 518	/* Note that config-io to empty slots may fail;
 519	 * they are empty when they don't have children.
 520	 * We will punt with the following conditions: Failure to get
 521	 * PE's state, EEH not support and Permanently unavailable
 522	 * state, PE is in good state.
 523	 */
 524	if ((ret < 0) ||
 525	    (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) {
 
 526		eeh_stats.false_positives++;
 527		pe->false_positives++;
 528		rc = 0;
 529		goto dn_unlock;
 530	}
 531
 532	/*
 533	 * It should be corner case that the parent PE has been
 534	 * put into frozen state as well. We should take care
 535	 * that at first.
 536	 */
 537	parent_pe = pe->parent;
 538	while (parent_pe) {
 539		/* Hit the ceiling ? */
 540		if (parent_pe->type & EEH_PE_PHB)
 541			break;
 542
 543		/* Frozen parent PE ? */
 544		ret = eeh_ops->get_state(parent_pe, NULL);
 545		if (ret > 0 && !eeh_state_active(ret))
 
 546			pe = parent_pe;
 547
 548		/* Next parent level */
 549		parent_pe = parent_pe->parent;
 550	}
 551
 552	eeh_stats.slot_resets++;
 553
 554	/* Avoid repeated reports of this failure, including problems
 555	 * with other functions on this device, and functions under
 556	 * bridges.
 557	 */
 558	eeh_pe_state_mark(pe, EEH_PE_ISOLATED);
 559	eeh_serialize_unlock(flags);
 560
 561	/* Most EEH events are due to device driver bugs.  Having
 562	 * a stack trace will help the device-driver authors figure
 563	 * out what happened.  So print that out.
 564	 */
 565	phb_pe = eeh_phb_pe_get(pe->phb);
 566	pr_err("EEH: Frozen PHB#%x-PE#%x detected\n",
 567	       pe->phb->global_number, pe->addr);
 568	pr_err("EEH: PE location: %s, PHB location: %s\n",
 569	       eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe));
 570	dump_stack();
 571
 572	eeh_send_failure_event(pe);
 573
 574	return 1;
 575
 576dn_unlock:
 577	eeh_serialize_unlock(flags);
 578	return rc;
 579}
 580
 581EXPORT_SYMBOL_GPL(eeh_dev_check_failure);
 582
 583/**
 584 * eeh_check_failure - Check if all 1's data is due to EEH slot freeze
 585 * @token: I/O address
 586 *
 587 * Check for an EEH failure at the given I/O address. Call this
 588 * routine if the result of a read was all 0xff's and you want to
 589 * find out if this is due to an EEH slot freeze event. This routine
 590 * will query firmware for the EEH status.
 591 *
 592 * Note this routine is safe to call in an interrupt context.
 593 */
 594int eeh_check_failure(const volatile void __iomem *token)
 595{
 596	unsigned long addr;
 597	struct eeh_dev *edev;
 598
 599	/* Finding the phys addr + pci device; this is pretty quick. */
 600	addr = eeh_token_to_phys((unsigned long __force) token);
 601	edev = eeh_addr_cache_get_dev(addr);
 602	if (!edev) {
 603		eeh_stats.no_device++;
 604		return 0;
 605	}
 606
 607	return eeh_dev_check_failure(edev);
 608}
 609EXPORT_SYMBOL(eeh_check_failure);
 610
 611
 612/**
 613 * eeh_pci_enable - Enable MMIO or DMA transfers for this slot
 614 * @pe: EEH PE
 615 *
 616 * This routine should be called to reenable frozen MMIO or DMA
 617 * so that it would work correctly again. It's useful while doing
 618 * recovery or log collection on the indicated device.
 619 */
 620int eeh_pci_enable(struct eeh_pe *pe, int function)
 621{
 622	int active_flag, rc;
 623
 624	/*
 625	 * pHyp doesn't allow to enable IO or DMA on unfrozen PE.
 626	 * Also, it's pointless to enable them on unfrozen PE. So
 627	 * we have to check before enabling IO or DMA.
 628	 */
 629	switch (function) {
 630	case EEH_OPT_THAW_MMIO:
 631		active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED;
 632		break;
 633	case EEH_OPT_THAW_DMA:
 634		active_flag = EEH_STATE_DMA_ACTIVE;
 635		break;
 636	case EEH_OPT_DISABLE:
 637	case EEH_OPT_ENABLE:
 638	case EEH_OPT_FREEZE_PE:
 639		active_flag = 0;
 640		break;
 641	default:
 642		pr_warn("%s: Invalid function %d\n",
 643			__func__, function);
 644		return -EINVAL;
 645	}
 646
 647	/*
 648	 * Check if IO or DMA has been enabled before
 649	 * enabling them.
 650	 */
 651	if (active_flag) {
 652		rc = eeh_ops->get_state(pe, NULL);
 653		if (rc < 0)
 654			return rc;
 655
 656		/* Needn't enable it at all */
 657		if (rc == EEH_STATE_NOT_SUPPORT)
 658			return 0;
 659
 660		/* It's already enabled */
 661		if (rc & active_flag)
 662			return 0;
 663	}
 664
 665
 666	/* Issue the request */
 667	rc = eeh_ops->set_option(pe, function);
 668	if (rc)
 669		pr_warn("%s: Unexpected state change %d on "
 670			"PHB#%x-PE#%x, err=%d\n",
 671			__func__, function, pe->phb->global_number,
 672			pe->addr, rc);
 673
 674	/* Check if the request is finished successfully */
 675	if (active_flag) {
 676		rc = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
 677		if (rc < 0)
 678			return rc;
 679
 680		if (rc & active_flag)
 681			return 0;
 682
 683		return -EIO;
 684	}
 685
 686	return rc;
 687}
 688
 689static void *eeh_disable_and_save_dev_state(void *data, void *userdata)
 690{
 691	struct eeh_dev *edev = data;
 692	struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
 693	struct pci_dev *dev = userdata;
 694
 695	/*
 696	 * The caller should have disabled and saved the
 697	 * state for the specified device
 698	 */
 699	if (!pdev || pdev == dev)
 700		return NULL;
 701
 702	/* Ensure we have D0 power state */
 703	pci_set_power_state(pdev, PCI_D0);
 704
 705	/* Save device state */
 706	pci_save_state(pdev);
 707
 708	/*
 709	 * Disable device to avoid any DMA traffic and
 710	 * interrupt from the device
 711	 */
 712	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
 713
 714	return NULL;
 715}
 716
 717static void *eeh_restore_dev_state(void *data, void *userdata)
 718{
 719	struct eeh_dev *edev = data;
 720	struct pci_dn *pdn = eeh_dev_to_pdn(edev);
 721	struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
 722	struct pci_dev *dev = userdata;
 723
 724	if (!pdev)
 725		return NULL;
 726
 727	/* Apply customization from firmware */
 728	if (pdn && eeh_ops->restore_config)
 729		eeh_ops->restore_config(pdn);
 730
 731	/* The caller should restore state for the specified device */
 732	if (pdev != dev)
 733		pci_restore_state(pdev);
 734
 735	return NULL;
 736}
 737
 738int eeh_restore_vf_config(struct pci_dn *pdn)
 739{
 740	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
 741	u32 devctl, cmd, cap2, aer_capctl;
 742	int old_mps;
 743
 744	if (edev->pcie_cap) {
 745		/* Restore MPS */
 746		old_mps = (ffs(pdn->mps) - 8) << 5;
 747		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
 748				     2, &devctl);
 749		devctl &= ~PCI_EXP_DEVCTL_PAYLOAD;
 750		devctl |= old_mps;
 751		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
 752				      2, devctl);
 753
 754		/* Disable Completion Timeout if possible */
 755		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2,
 756				     4, &cap2);
 757		if (cap2 & PCI_EXP_DEVCAP2_COMP_TMOUT_DIS) {
 758			eeh_ops->read_config(pdn,
 759					     edev->pcie_cap + PCI_EXP_DEVCTL2,
 760					     4, &cap2);
 761			cap2 |= PCI_EXP_DEVCTL2_COMP_TMOUT_DIS;
 762			eeh_ops->write_config(pdn,
 763					      edev->pcie_cap + PCI_EXP_DEVCTL2,
 764					      4, cap2);
 765		}
 766	}
 767
 768	/* Enable SERR and parity checking */
 769	eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd);
 770	cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR);
 771	eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd);
 772
 773	/* Enable report various errors */
 774	if (edev->pcie_cap) {
 775		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
 776				     2, &devctl);
 777		devctl &= ~PCI_EXP_DEVCTL_CERE;
 778		devctl |= (PCI_EXP_DEVCTL_NFERE |
 779			   PCI_EXP_DEVCTL_FERE |
 780			   PCI_EXP_DEVCTL_URRE);
 781		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
 782				      2, devctl);
 783	}
 784
 785	/* Enable ECRC generation and check */
 786	if (edev->pcie_cap && edev->aer_cap) {
 787		eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP,
 788				     4, &aer_capctl);
 789		aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
 790		eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP,
 791				      4, aer_capctl);
 792	}
 793
 794	return 0;
 795}
 796
 797/**
 798 * pcibios_set_pcie_reset_state - Set PCI-E reset state
 799 * @dev: pci device struct
 800 * @state: reset state to enter
 801 *
 802 * Return value:
 803 * 	0 if success
 804 */
 805int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
 806{
 807	struct eeh_dev *edev = pci_dev_to_eeh_dev(dev);
 808	struct eeh_pe *pe = eeh_dev_to_pe(edev);
 809
 810	if (!pe) {
 811		pr_err("%s: No PE found on PCI device %s\n",
 812			__func__, pci_name(dev));
 813		return -EINVAL;
 814	}
 815
 816	switch (state) {
 817	case pcie_deassert_reset:
 818		eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
 819		eeh_unfreeze_pe(pe, false);
 820		if (!(pe->type & EEH_PE_VF))
 821			eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
 822		eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev);
 823		eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
 824		break;
 825	case pcie_hot_reset:
 826		eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED);
 827		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
 828		eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
 829		if (!(pe->type & EEH_PE_VF))
 830			eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
 831		eeh_ops->reset(pe, EEH_RESET_HOT);
 832		break;
 833	case pcie_warm_reset:
 834		eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED);
 835		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
 836		eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
 837		if (!(pe->type & EEH_PE_VF))
 838			eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
 839		eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);
 840		break;
 841	default:
 842		eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED);
 843		return -EINVAL;
 844	};
 845
 846	return 0;
 847}
 848
 849/**
 850 * eeh_set_pe_freset - Check the required reset for the indicated device
 851 * @data: EEH device
 852 * @flag: return value
 853 *
 854 * Each device might have its preferred reset type: fundamental or
 855 * hot reset. The routine is used to collected the information for
 856 * the indicated device and its children so that the bunch of the
 857 * devices could be reset properly.
 858 */
 859static void *eeh_set_dev_freset(void *data, void *flag)
 860{
 861	struct pci_dev *dev;
 862	unsigned int *freset = (unsigned int *)flag;
 863	struct eeh_dev *edev = (struct eeh_dev *)data;
 864
 865	dev = eeh_dev_to_pci_dev(edev);
 866	if (dev)
 867		*freset |= dev->needs_freset;
 868
 869	return NULL;
 870}
 871
 872/**
 873 * eeh_pe_reset_full - Complete a full reset process on the indicated PE
 874 * @pe: EEH PE
 875 *
 876 * This function executes a full reset procedure on a PE, including setting
 877 * the appropriate flags, performing a fundamental or hot reset, and then
 878 * deactivating the reset status.  It is designed to be used within the EEH
 879 * subsystem, as opposed to eeh_pe_reset which is exported to drivers and
 880 * only performs a single operation at a time.
 881 *
 882 * This function will attempt to reset a PE three times before failing.
 883 */
 884int eeh_pe_reset_full(struct eeh_pe *pe)
 885{
 
 886	int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
 887	int type = EEH_RESET_HOT;
 888	unsigned int freset = 0;
 889	int i, state, ret;
 890
 891	/*
 892	 * Determine the type of reset to perform - hot or fundamental.
 893	 * Hot reset is the default operation, unless any device under the
 894	 * PE requires a fundamental reset.
 895	 */
 896	eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset);
 897
 898	if (freset)
 899		type = EEH_RESET_FUNDAMENTAL;
 900
 901	/* Mark the PE as in reset state and block config space accesses */
 902	eeh_pe_state_mark(pe, reset_state);
 903
 904	/* Make three attempts at resetting the bus */
 905	for (i = 0; i < 3; i++) {
 906		ret = eeh_pe_reset(pe, type);
 907		if (ret)
 908			break;
 909
 910		ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
 911		if (ret)
 912			break;
 913
 914		/* Wait until the PE is in a functioning state */
 915		state = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
 916		if (eeh_state_active(state))
 917			break;
 918
 919		if (state < 0) {
 920			pr_warn("%s: Unrecoverable slot failure on PHB#%x-PE#%x",
 921				__func__, pe->phb->global_number, pe->addr);
 922			ret = -ENOTRECOVERABLE;
 923			break;
 924		}
 925
 926		/* Set error in case this is our last attempt */
 927		ret = -EIO;
 928		pr_warn("%s: Failure %d resetting PHB#%x-PE#%x\n (%d)\n",
 929			__func__, state, pe->phb->global_number, pe->addr, (i + 1));
 930	}
 931
 932	eeh_pe_state_clear(pe, reset_state);
 933	return ret;
 934}
 935
 936/**
 937 * eeh_save_bars - Save device bars
 938 * @edev: PCI device associated EEH device
 939 *
 940 * Save the values of the device bars. Unlike the restore
 941 * routine, this routine is *not* recursive. This is because
 942 * PCI devices are added individually; but, for the restore,
 943 * an entire slot is reset at a time.
 944 */
 945void eeh_save_bars(struct eeh_dev *edev)
 946{
 947	struct pci_dn *pdn;
 948	int i;
 949
 950	pdn = eeh_dev_to_pdn(edev);
 951	if (!pdn)
 952		return;
 953
 954	for (i = 0; i < 16; i++)
 955		eeh_ops->read_config(pdn, i * 4, 4, &edev->config_space[i]);
 956
 957	/*
 958	 * For PCI bridges including root port, we need enable bus
 959	 * master explicitly. Otherwise, it can't fetch IODA table
 960	 * entries correctly. So we cache the bit in advance so that
 961	 * we can restore it after reset, either PHB range or PE range.
 962	 */
 963	if (edev->mode & EEH_DEV_BRIDGE)
 964		edev->config_space[1] |= PCI_COMMAND_MASTER;
 965}
 966
 967/**
 968 * eeh_ops_register - Register platform dependent EEH operations
 969 * @ops: platform dependent EEH operations
 970 *
 971 * Register the platform dependent EEH operation callback
 972 * functions. The platform should call this function before
 973 * any other EEH operations.
 974 */
 975int __init eeh_ops_register(struct eeh_ops *ops)
 976{
 977	if (!ops->name) {
 978		pr_warn("%s: Invalid EEH ops name for %p\n",
 979			__func__, ops);
 980		return -EINVAL;
 981	}
 982
 983	if (eeh_ops && eeh_ops != ops) {
 984		pr_warn("%s: EEH ops of platform %s already existing (%s)\n",
 985			__func__, eeh_ops->name, ops->name);
 986		return -EEXIST;
 987	}
 988
 989	eeh_ops = ops;
 990
 991	return 0;
 992}
 993
 994/**
 995 * eeh_ops_unregister - Unreigster platform dependent EEH operations
 996 * @name: name of EEH platform operations
 997 *
 998 * Unregister the platform dependent EEH operation callback
 999 * functions.
1000 */
1001int __exit eeh_ops_unregister(const char *name)
1002{
1003	if (!name || !strlen(name)) {
1004		pr_warn("%s: Invalid EEH ops name\n",
1005			__func__);
1006		return -EINVAL;
1007	}
1008
1009	if (eeh_ops && !strcmp(eeh_ops->name, name)) {
1010		eeh_ops = NULL;
1011		return 0;
1012	}
1013
1014	return -EEXIST;
1015}
1016
1017static int eeh_reboot_notifier(struct notifier_block *nb,
1018			       unsigned long action, void *unused)
1019{
1020	eeh_clear_flag(EEH_ENABLED);
1021	return NOTIFY_DONE;
1022}
1023
1024static struct notifier_block eeh_reboot_nb = {
1025	.notifier_call = eeh_reboot_notifier,
1026};
1027
1028void eeh_probe_devices(void)
1029{
1030	struct pci_controller *hose, *tmp;
1031	struct pci_dn *pdn;
1032
1033	/* Enable EEH for all adapters */
1034	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1035		pdn = hose->pci_data;
1036		traverse_pci_dn(pdn, eeh_ops->probe, NULL);
1037	}
1038}
1039
1040/**
1041 * eeh_init - EEH initialization
1042 *
1043 * Initialize EEH by trying to enable it for all of the adapters in the system.
1044 * As a side effect we can determine here if eeh is supported at all.
1045 * Note that we leave EEH on so failed config cycles won't cause a machine
1046 * check.  If a user turns off EEH for a particular adapter they are really
1047 * telling Linux to ignore errors.  Some hardware (e.g. POWER5) won't
1048 * grant access to a slot if EEH isn't enabled, and so we always enable
1049 * EEH for all slots/all devices.
1050 *
1051 * The eeh-force-off option disables EEH checking globally, for all slots.
1052 * Even if force-off is set, the EEH hardware is still enabled, so that
1053 * newer systems can boot.
1054 */
1055static int eeh_init(void)
1056{
1057	struct pci_controller *hose, *tmp;
 
 
1058	int ret = 0;
1059
 
 
 
 
 
 
 
 
 
1060	/* Register reboot notifier */
1061	ret = register_reboot_notifier(&eeh_reboot_nb);
1062	if (ret) {
1063		pr_warn("%s: Failed to register notifier (%d)\n",
1064			__func__, ret);
1065		return ret;
1066	}
1067
1068	/* call platform initialization function */
1069	if (!eeh_ops) {
1070		pr_warn("%s: Platform EEH operation not found\n",
1071			__func__);
1072		return -EEXIST;
1073	} else if ((ret = eeh_ops->init()))
1074		return ret;
1075
1076	/* Initialize PHB PEs */
1077	list_for_each_entry_safe(hose, tmp, &hose_list, list_node)
1078		eeh_dev_phb_init_dynamic(hose);
1079
1080	/* Initialize EEH event */
1081	ret = eeh_event_init();
1082	if (ret)
1083		return ret;
1084
1085	eeh_probe_devices();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1086
1087	if (eeh_enabled())
1088		pr_info("EEH: PCI Enhanced I/O Error Handling Enabled\n");
1089	else
1090		pr_info("EEH: No capable adapters found\n");
1091
1092	return ret;
1093}
1094
1095core_initcall_sync(eeh_init);
1096
1097/**
1098 * eeh_add_device_early - Enable EEH for the indicated device node
1099 * @pdn: PCI device node for which to set up EEH
1100 *
1101 * This routine must be used to perform EEH initialization for PCI
1102 * devices that were added after system boot (e.g. hotplug, dlpar).
1103 * This routine must be called before any i/o is performed to the
1104 * adapter (inluding any config-space i/o).
1105 * Whether this actually enables EEH or not for this device depends
1106 * on the CEC architecture, type of the device, on earlier boot
1107 * command-line arguments & etc.
1108 */
1109void eeh_add_device_early(struct pci_dn *pdn)
1110{
1111	struct pci_controller *phb = pdn ? pdn->phb : NULL;
1112	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
1113
1114	if (!edev)
1115		return;
1116
1117	if (!eeh_has_flag(EEH_PROBE_MODE_DEVTREE))
1118		return;
1119
1120	/* USB Bus children of PCI devices will not have BUID's */
 
1121	if (NULL == phb ||
1122	    (eeh_has_flag(EEH_PROBE_MODE_DEVTREE) && 0 == phb->buid))
1123		return;
1124
1125	eeh_ops->probe(pdn, NULL);
1126}
1127
1128/**
1129 * eeh_add_device_tree_early - Enable EEH for the indicated device
1130 * @pdn: PCI device node
1131 *
1132 * This routine must be used to perform EEH initialization for the
1133 * indicated PCI device that was added after system boot (e.g.
1134 * hotplug, dlpar).
1135 */
1136void eeh_add_device_tree_early(struct pci_dn *pdn)
1137{
1138	struct pci_dn *n;
1139
1140	if (!pdn)
1141		return;
1142
1143	list_for_each_entry(n, &pdn->child_list, list)
1144		eeh_add_device_tree_early(n);
1145	eeh_add_device_early(pdn);
1146}
1147EXPORT_SYMBOL_GPL(eeh_add_device_tree_early);
1148
1149/**
1150 * eeh_add_device_late - Perform EEH initialization for the indicated pci device
1151 * @dev: pci device for which to set up EEH
1152 *
1153 * This routine must be used to complete EEH initialization for PCI
1154 * devices that were added after system boot (e.g. hotplug, dlpar).
1155 */
1156void eeh_add_device_late(struct pci_dev *dev)
1157{
1158	struct pci_dn *pdn;
1159	struct eeh_dev *edev;
1160
1161	if (!dev || !eeh_enabled())
1162		return;
1163
1164	pr_debug("EEH: Adding device %s\n", pci_name(dev));
1165
1166	pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn);
1167	edev = pdn_to_eeh_dev(pdn);
1168	if (edev->pdev == dev) {
1169		pr_debug("EEH: Already referenced !\n");
1170		return;
1171	}
1172
1173	/*
1174	 * The EEH cache might not be removed correctly because of
1175	 * unbalanced kref to the device during unplug time, which
1176	 * relies on pcibios_release_device(). So we have to remove
1177	 * that here explicitly.
1178	 */
1179	if (edev->pdev) {
1180		eeh_rmv_from_parent_pe(edev);
1181		eeh_addr_cache_rmv_dev(edev->pdev);
1182		eeh_sysfs_remove_device(edev->pdev);
1183		edev->mode &= ~EEH_DEV_SYSFS;
1184
1185		/*
1186		 * We definitely should have the PCI device removed
1187		 * though it wasn't correctly. So we needn't call
1188		 * into error handler afterwards.
1189		 */
1190		edev->mode |= EEH_DEV_NO_HANDLER;
1191
1192		edev->pdev = NULL;
1193		dev->dev.archdata.edev = NULL;
1194	}
1195
1196	if (eeh_has_flag(EEH_PROBE_MODE_DEV))
1197		eeh_ops->probe(pdn, NULL);
1198
1199	edev->pdev = dev;
1200	dev->dev.archdata.edev = edev;
1201
1202	eeh_addr_cache_insert_dev(dev);
1203}
1204
1205/**
1206 * eeh_add_device_tree_late - Perform EEH initialization for the indicated PCI bus
1207 * @bus: PCI bus
1208 *
1209 * This routine must be used to perform EEH initialization for PCI
1210 * devices which are attached to the indicated PCI bus. The PCI bus
1211 * is added after system boot through hotplug or dlpar.
1212 */
1213void eeh_add_device_tree_late(struct pci_bus *bus)
1214{
1215	struct pci_dev *dev;
1216
1217	list_for_each_entry(dev, &bus->devices, bus_list) {
1218		eeh_add_device_late(dev);
1219		if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1220			struct pci_bus *subbus = dev->subordinate;
1221			if (subbus)
1222				eeh_add_device_tree_late(subbus);
1223		}
1224	}
1225}
1226EXPORT_SYMBOL_GPL(eeh_add_device_tree_late);
1227
1228/**
1229 * eeh_add_sysfs_files - Add EEH sysfs files for the indicated PCI bus
1230 * @bus: PCI bus
1231 *
1232 * This routine must be used to add EEH sysfs files for PCI
1233 * devices which are attached to the indicated PCI bus. The PCI bus
1234 * is added after system boot through hotplug or dlpar.
1235 */
1236void eeh_add_sysfs_files(struct pci_bus *bus)
1237{
1238	struct pci_dev *dev;
1239
1240	list_for_each_entry(dev, &bus->devices, bus_list) {
1241		eeh_sysfs_add_device(dev);
1242		if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1243			struct pci_bus *subbus = dev->subordinate;
1244			if (subbus)
1245				eeh_add_sysfs_files(subbus);
1246		}
1247	}
1248}
1249EXPORT_SYMBOL_GPL(eeh_add_sysfs_files);
1250
1251/**
1252 * eeh_remove_device - Undo EEH setup for the indicated pci device
1253 * @dev: pci device to be removed
1254 *
1255 * This routine should be called when a device is removed from
1256 * a running system (e.g. by hotplug or dlpar).  It unregisters
1257 * the PCI device from the EEH subsystem.  I/O errors affecting
1258 * this device will no longer be detected after this call; thus,
1259 * i/o errors affecting this slot may leave this device unusable.
1260 */
1261void eeh_remove_device(struct pci_dev *dev)
1262{
1263	struct eeh_dev *edev;
1264
1265	if (!dev || !eeh_enabled())
1266		return;
1267	edev = pci_dev_to_eeh_dev(dev);
1268
1269	/* Unregister the device with the EEH/PCI address search system */
1270	pr_debug("EEH: Removing device %s\n", pci_name(dev));
1271
1272	if (!edev || !edev->pdev || !edev->pe) {
1273		pr_debug("EEH: Not referenced !\n");
1274		return;
1275	}
1276
1277	/*
1278	 * During the hotplug for EEH error recovery, we need the EEH
1279	 * device attached to the parent PE in order for BAR restore
1280	 * a bit later. So we keep it for BAR restore and remove it
1281	 * from the parent PE during the BAR resotre.
1282	 */
1283	edev->pdev = NULL;
1284
1285	/*
1286	 * The flag "in_error" is used to trace EEH devices for VFs
1287	 * in error state or not. It's set in eeh_report_error(). If
1288	 * it's not set, eeh_report_{reset,resume}() won't be called
1289	 * for the VF EEH device.
1290	 */
1291	edev->in_error = false;
1292	dev->dev.archdata.edev = NULL;
1293	if (!(edev->pe->state & EEH_PE_KEEP))
1294		eeh_rmv_from_parent_pe(edev);
1295	else
1296		edev->mode |= EEH_DEV_DISCONNECTED;
1297
1298	/*
1299	 * We're removing from the PCI subsystem, that means
1300	 * the PCI device driver can't support EEH or not
1301	 * well. So we rely on hotplug completely to do recovery
1302	 * for the specific PCI device.
1303	 */
1304	edev->mode |= EEH_DEV_NO_HANDLER;
1305
1306	eeh_addr_cache_rmv_dev(dev);
1307	eeh_sysfs_remove_device(dev);
1308	edev->mode &= ~EEH_DEV_SYSFS;
1309}
1310
1311int eeh_unfreeze_pe(struct eeh_pe *pe, bool sw_state)
1312{
1313	int ret;
1314
1315	ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
1316	if (ret) {
1317		pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n",
1318			__func__, ret, pe->phb->global_number, pe->addr);
1319		return ret;
1320	}
1321
1322	ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA);
1323	if (ret) {
1324		pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n",
1325			__func__, ret, pe->phb->global_number, pe->addr);
1326		return ret;
1327	}
1328
1329	/* Clear software isolated state */
1330	if (sw_state && (pe->state & EEH_PE_ISOLATED))
1331		eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
1332
1333	return ret;
1334}
1335
1336
1337static struct pci_device_id eeh_reset_ids[] = {
1338	{ PCI_DEVICE(0x19a2, 0x0710) },	/* Emulex, BE     */
1339	{ PCI_DEVICE(0x10df, 0xe220) },	/* Emulex, Lancer */
1340	{ PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */
1341	{ 0 }
1342};
1343
1344static int eeh_pe_change_owner(struct eeh_pe *pe)
1345{
1346	struct eeh_dev *edev, *tmp;
1347	struct pci_dev *pdev;
1348	struct pci_device_id *id;
1349	int ret;
1350
1351	/* Check PE state */
 
1352	ret = eeh_ops->get_state(pe, NULL);
1353	if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT)
1354		return 0;
1355
1356	/* Unfrozen PE, nothing to do */
1357	if (eeh_state_active(ret))
1358		return 0;
1359
1360	/* Frozen PE, check if it needs PE level reset */
1361	eeh_pe_for_each_dev(pe, edev, tmp) {
1362		pdev = eeh_dev_to_pci_dev(edev);
1363		if (!pdev)
1364			continue;
1365
1366		for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) {
1367			if (id->vendor != PCI_ANY_ID &&
1368			    id->vendor != pdev->vendor)
1369				continue;
1370			if (id->device != PCI_ANY_ID &&
1371			    id->device != pdev->device)
1372				continue;
1373			if (id->subvendor != PCI_ANY_ID &&
1374			    id->subvendor != pdev->subsystem_vendor)
1375				continue;
1376			if (id->subdevice != PCI_ANY_ID &&
1377			    id->subdevice != pdev->subsystem_device)
1378				continue;
1379
1380			return eeh_pe_reset_and_recover(pe);
1381		}
1382	}
1383
1384	return eeh_unfreeze_pe(pe, true);
1385}
1386
1387/**
1388 * eeh_dev_open - Increase count of pass through devices for PE
1389 * @pdev: PCI device
1390 *
1391 * Increase count of passed through devices for the indicated
1392 * PE. In the result, the EEH errors detected on the PE won't be
1393 * reported. The PE owner will be responsible for detection
1394 * and recovery.
1395 */
1396int eeh_dev_open(struct pci_dev *pdev)
1397{
1398	struct eeh_dev *edev;
1399	int ret = -ENODEV;
1400
1401	mutex_lock(&eeh_dev_mutex);
1402
1403	/* No PCI device ? */
1404	if (!pdev)
1405		goto out;
1406
1407	/* No EEH device or PE ? */
1408	edev = pci_dev_to_eeh_dev(pdev);
1409	if (!edev || !edev->pe)
1410		goto out;
1411
1412	/*
1413	 * The PE might have been put into frozen state, but we
1414	 * didn't detect that yet. The passed through PCI devices
1415	 * in frozen PE won't work properly. Clear the frozen state
1416	 * in advance.
1417	 */
1418	ret = eeh_pe_change_owner(edev->pe);
1419	if (ret)
1420		goto out;
1421
1422	/* Increase PE's pass through count */
1423	atomic_inc(&edev->pe->pass_dev_cnt);
1424	mutex_unlock(&eeh_dev_mutex);
1425
1426	return 0;
1427out:
1428	mutex_unlock(&eeh_dev_mutex);
1429	return ret;
1430}
1431EXPORT_SYMBOL_GPL(eeh_dev_open);
1432
1433/**
1434 * eeh_dev_release - Decrease count of pass through devices for PE
1435 * @pdev: PCI device
1436 *
1437 * Decrease count of pass through devices for the indicated PE. If
1438 * there is no passed through device in PE, the EEH errors detected
1439 * on the PE will be reported and handled as usual.
1440 */
1441void eeh_dev_release(struct pci_dev *pdev)
1442{
1443	struct eeh_dev *edev;
1444
1445	mutex_lock(&eeh_dev_mutex);
1446
1447	/* No PCI device ? */
1448	if (!pdev)
1449		goto out;
1450
1451	/* No EEH device ? */
1452	edev = pci_dev_to_eeh_dev(pdev);
1453	if (!edev || !edev->pe || !eeh_pe_passed(edev->pe))
1454		goto out;
1455
1456	/* Decrease PE's pass through count */
1457	WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0);
1458	eeh_pe_change_owner(edev->pe);
1459out:
1460	mutex_unlock(&eeh_dev_mutex);
1461}
1462EXPORT_SYMBOL(eeh_dev_release);
1463
1464#ifdef CONFIG_IOMMU_API
1465
1466static int dev_has_iommu_table(struct device *dev, void *data)
1467{
1468	struct pci_dev *pdev = to_pci_dev(dev);
1469	struct pci_dev **ppdev = data;
1470
1471	if (!dev)
1472		return 0;
1473
1474	if (dev->iommu_group) {
1475		*ppdev = pdev;
1476		return 1;
1477	}
1478
1479	return 0;
1480}
1481
1482/**
1483 * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE
1484 * @group: IOMMU group
1485 *
1486 * The routine is called to convert IOMMU group to EEH PE.
1487 */
1488struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group)
1489{
1490	struct pci_dev *pdev = NULL;
1491	struct eeh_dev *edev;
1492	int ret;
1493
1494	/* No IOMMU group ? */
1495	if (!group)
1496		return NULL;
1497
1498	ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table);
1499	if (!ret || !pdev)
1500		return NULL;
1501
1502	/* No EEH device or PE ? */
1503	edev = pci_dev_to_eeh_dev(pdev);
1504	if (!edev || !edev->pe)
1505		return NULL;
1506
1507	return edev->pe;
1508}
1509EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe);
1510
1511#endif /* CONFIG_IOMMU_API */
1512
1513/**
1514 * eeh_pe_set_option - Set options for the indicated PE
1515 * @pe: EEH PE
1516 * @option: requested option
1517 *
1518 * The routine is called to enable or disable EEH functionality
1519 * on the indicated PE, to enable IO or DMA for the frozen PE.
1520 */
1521int eeh_pe_set_option(struct eeh_pe *pe, int option)
1522{
1523	int ret = 0;
1524
1525	/* Invalid PE ? */
1526	if (!pe)
1527		return -ENODEV;
1528
1529	/*
1530	 * EEH functionality could possibly be disabled, just
1531	 * return error for the case. And the EEH functinality
1532	 * isn't expected to be disabled on one specific PE.
1533	 */
1534	switch (option) {
1535	case EEH_OPT_ENABLE:
1536		if (eeh_enabled()) {
1537			ret = eeh_pe_change_owner(pe);
1538			break;
1539		}
1540		ret = -EIO;
1541		break;
1542	case EEH_OPT_DISABLE:
1543		break;
1544	case EEH_OPT_THAW_MMIO:
1545	case EEH_OPT_THAW_DMA:
1546	case EEH_OPT_FREEZE_PE:
1547		if (!eeh_ops || !eeh_ops->set_option) {
1548			ret = -ENOENT;
1549			break;
1550		}
1551
1552		ret = eeh_pci_enable(pe, option);
1553		break;
1554	default:
1555		pr_debug("%s: Option %d out of range (%d, %d)\n",
1556			__func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA);
1557		ret = -EINVAL;
1558	}
1559
1560	return ret;
1561}
1562EXPORT_SYMBOL_GPL(eeh_pe_set_option);
1563
1564/**
1565 * eeh_pe_get_state - Retrieve PE's state
1566 * @pe: EEH PE
1567 *
1568 * Retrieve the PE's state, which includes 3 aspects: enabled
1569 * DMA, enabled IO and asserted reset.
1570 */
1571int eeh_pe_get_state(struct eeh_pe *pe)
1572{
1573	int result, ret = 0;
1574	bool rst_active, dma_en, mmio_en;
1575
1576	/* Existing PE ? */
1577	if (!pe)
1578		return -ENODEV;
1579
1580	if (!eeh_ops || !eeh_ops->get_state)
1581		return -ENOENT;
1582
1583	/*
1584	 * If the parent PE is owned by the host kernel and is undergoing
1585	 * error recovery, we should return the PE state as temporarily
1586	 * unavailable so that the error recovery on the guest is suspended
1587	 * until the recovery completes on the host.
1588	 */
1589	if (pe->parent &&
1590	    !(pe->state & EEH_PE_REMOVED) &&
1591	    (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING)))
1592		return EEH_PE_STATE_UNAVAIL;
1593
1594	result = eeh_ops->get_state(pe, NULL);
1595	rst_active = !!(result & EEH_STATE_RESET_ACTIVE);
1596	dma_en = !!(result & EEH_STATE_DMA_ENABLED);
1597	mmio_en = !!(result & EEH_STATE_MMIO_ENABLED);
1598
1599	if (rst_active)
1600		ret = EEH_PE_STATE_RESET;
1601	else if (dma_en && mmio_en)
1602		ret = EEH_PE_STATE_NORMAL;
1603	else if (!dma_en && !mmio_en)
1604		ret = EEH_PE_STATE_STOPPED_IO_DMA;
1605	else if (!dma_en && mmio_en)
1606		ret = EEH_PE_STATE_STOPPED_DMA;
1607	else
1608		ret = EEH_PE_STATE_UNAVAIL;
1609
1610	return ret;
1611}
1612EXPORT_SYMBOL_GPL(eeh_pe_get_state);
1613
1614static int eeh_pe_reenable_devices(struct eeh_pe *pe)
1615{
1616	struct eeh_dev *edev, *tmp;
1617	struct pci_dev *pdev;
1618	int ret = 0;
1619
1620	/* Restore config space */
1621	eeh_pe_restore_bars(pe);
1622
1623	/*
1624	 * Reenable PCI devices as the devices passed
1625	 * through are always enabled before the reset.
1626	 */
1627	eeh_pe_for_each_dev(pe, edev, tmp) {
1628		pdev = eeh_dev_to_pci_dev(edev);
1629		if (!pdev)
1630			continue;
1631
1632		ret = pci_reenable_device(pdev);
1633		if (ret) {
1634			pr_warn("%s: Failure %d reenabling %s\n",
1635				__func__, ret, pci_name(pdev));
1636			return ret;
1637		}
1638	}
1639
1640	/* The PE is still in frozen state */
1641	return eeh_unfreeze_pe(pe, true);
1642}
1643
1644
1645/**
1646 * eeh_pe_reset - Issue PE reset according to specified type
1647 * @pe: EEH PE
1648 * @option: reset type
1649 *
1650 * The routine is called to reset the specified PE with the
1651 * indicated type, either fundamental reset or hot reset.
1652 * PE reset is the most important part for error recovery.
1653 */
1654int eeh_pe_reset(struct eeh_pe *pe, int option)
1655{
1656	int ret = 0;
1657
1658	/* Invalid PE ? */
1659	if (!pe)
1660		return -ENODEV;
1661
1662	if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset)
1663		return -ENOENT;
1664
1665	switch (option) {
1666	case EEH_RESET_DEACTIVATE:
1667		ret = eeh_ops->reset(pe, option);
1668		eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
1669		if (ret)
1670			break;
1671
1672		ret = eeh_pe_reenable_devices(pe);
1673		break;
1674	case EEH_RESET_HOT:
1675	case EEH_RESET_FUNDAMENTAL:
1676		/*
1677		 * Proactively freeze the PE to drop all MMIO access
1678		 * during reset, which should be banned as it's always
1679		 * cause recursive EEH error.
1680		 */
1681		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
1682
1683		eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
1684		ret = eeh_ops->reset(pe, option);
1685		break;
1686	default:
1687		pr_debug("%s: Unsupported option %d\n",
1688			__func__, option);
1689		ret = -EINVAL;
1690	}
1691
1692	return ret;
1693}
1694EXPORT_SYMBOL_GPL(eeh_pe_reset);
1695
1696/**
1697 * eeh_pe_configure - Configure PCI bridges after PE reset
1698 * @pe: EEH PE
1699 *
1700 * The routine is called to restore the PCI config space for
1701 * those PCI devices, especially PCI bridges affected by PE
1702 * reset issued previously.
1703 */
1704int eeh_pe_configure(struct eeh_pe *pe)
1705{
1706	int ret = 0;
1707
1708	/* Invalid PE ? */
1709	if (!pe)
1710		return -ENODEV;
1711
1712	return ret;
1713}
1714EXPORT_SYMBOL_GPL(eeh_pe_configure);
1715
1716/**
1717 * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE
1718 * @pe: the indicated PE
1719 * @type: error type
1720 * @function: error function
1721 * @addr: address
1722 * @mask: address mask
1723 *
1724 * The routine is called to inject the specified PCI error, which
1725 * is determined by @type and @function, to the indicated PE for
1726 * testing purpose.
1727 */
1728int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func,
1729		      unsigned long addr, unsigned long mask)
1730{
1731	/* Invalid PE ? */
1732	if (!pe)
1733		return -ENODEV;
1734
1735	/* Unsupported operation ? */
1736	if (!eeh_ops || !eeh_ops->err_inject)
1737		return -ENOENT;
1738
1739	/* Check on PCI error type */
1740	if (type != EEH_ERR_TYPE_32 && type != EEH_ERR_TYPE_64)
1741		return -EINVAL;
1742
1743	/* Check on PCI error function */
1744	if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX)
1745		return -EINVAL;
1746
1747	return eeh_ops->err_inject(pe, type, func, addr, mask);
1748}
1749EXPORT_SYMBOL_GPL(eeh_pe_inject_err);
1750
1751static int proc_eeh_show(struct seq_file *m, void *v)
1752{
1753	if (!eeh_enabled()) {
1754		seq_printf(m, "EEH Subsystem is globally disabled\n");
1755		seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs);
1756	} else {
1757		seq_printf(m, "EEH Subsystem is enabled\n");
1758		seq_printf(m,
1759				"no device=%llu\n"
1760				"no device node=%llu\n"
1761				"no config address=%llu\n"
1762				"check not wanted=%llu\n"
1763				"eeh_total_mmio_ffs=%llu\n"
1764				"eeh_false_positives=%llu\n"
1765				"eeh_slot_resets=%llu\n",
1766				eeh_stats.no_device,
1767				eeh_stats.no_dn,
1768				eeh_stats.no_cfg_addr,
1769				eeh_stats.ignored_check,
1770				eeh_stats.total_mmio_ffs,
1771				eeh_stats.false_positives,
1772				eeh_stats.slot_resets);
1773	}
1774
1775	return 0;
1776}
1777
1778static int proc_eeh_open(struct inode *inode, struct file *file)
1779{
1780	return single_open(file, proc_eeh_show, NULL);
1781}
1782
1783static const struct file_operations proc_eeh_operations = {
1784	.open      = proc_eeh_open,
1785	.read      = seq_read,
1786	.llseek    = seq_lseek,
1787	.release   = single_release,
1788};
1789
1790#ifdef CONFIG_DEBUG_FS
1791static int eeh_enable_dbgfs_set(void *data, u64 val)
1792{
1793	if (val)
1794		eeh_clear_flag(EEH_FORCE_DISABLED);
1795	else
1796		eeh_add_flag(EEH_FORCE_DISABLED);
 
 
 
 
1797
1798	return 0;
1799}
1800
1801static int eeh_enable_dbgfs_get(void *data, u64 *val)
1802{
1803	if (eeh_enabled())
1804		*val = 0x1ul;
1805	else
1806		*val = 0x0ul;
1807	return 0;
1808}
1809
1810static int eeh_freeze_dbgfs_set(void *data, u64 val)
1811{
1812	eeh_max_freezes = val;
1813	return 0;
1814}
1815
1816static int eeh_freeze_dbgfs_get(void *data, u64 *val)
1817{
1818	*val = eeh_max_freezes;
1819	return 0;
1820}
1821
1822DEFINE_SIMPLE_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get,
1823			eeh_enable_dbgfs_set, "0x%llx\n");
1824DEFINE_SIMPLE_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get,
1825			eeh_freeze_dbgfs_set, "0x%llx\n");
1826#endif
1827
1828static int __init eeh_init_proc(void)
1829{
1830	if (machine_is(pseries) || machine_is(powernv)) {
1831		proc_create("powerpc/eeh", 0, NULL, &proc_eeh_operations);
1832#ifdef CONFIG_DEBUG_FS
1833		debugfs_create_file("eeh_enable", 0600,
1834                                    powerpc_debugfs_root, NULL,
1835                                    &eeh_enable_dbgfs_ops);
1836		debugfs_create_file("eeh_max_freezes", 0600,
1837				    powerpc_debugfs_root, NULL,
1838				    &eeh_freeze_dbgfs_ops);
1839#endif
1840	}
1841
1842	return 0;
1843}
1844__initcall(eeh_init_proc);