1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PCI Bus Services, see include/linux/pci.h for further explanation.
   4 *
   5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
   6 * David Mosberger-Tang
   7 *
   8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
   9 */
  10
  11#include <linux/acpi.h>
  12#include <linux/kernel.h>
  13#include <linux/delay.h>
  14#include <linux/dmi.h>
  15#include <linux/init.h>
  16#include <linux/msi.h>
  17#include <linux/of.h>
  18#include <linux/pci.h>
  19#include <linux/pm.h>
  20#include <linux/slab.h>
  21#include <linux/module.h>
  22#include <linux/spinlock.h>
  23#include <linux/string.h>
  24#include <linux/log2.h>
  25#include <linux/logic_pio.h>
  26#include <linux/pm_wakeup.h>
 
  27#include <linux/device.h>
  28#include <linux/pm_runtime.h>
  29#include <linux/pci_hotplug.h>
  30#include <linux/vmalloc.h>
  31#include <asm/dma.h>
  32#include <linux/aer.h>
  33#include <linux/bitfield.h>
  34#include "pci.h"
  35
  36DEFINE_MUTEX(pci_slot_mutex);
  37
  38const char *pci_power_names[] = {
  39	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
  40};
  41EXPORT_SYMBOL_GPL(pci_power_names);
  42
  43#ifdef CONFIG_X86_32
  44int isa_dma_bridge_buggy;
  45EXPORT_SYMBOL(isa_dma_bridge_buggy);
  46#endif
  47
  48int pci_pci_problems;
  49EXPORT_SYMBOL(pci_pci_problems);
  50
  51unsigned int pci_pm_d3hot_delay;
  52
  53static void pci_pme_list_scan(struct work_struct *work);
  54
  55static LIST_HEAD(pci_pme_list);
  56static DEFINE_MUTEX(pci_pme_list_mutex);
  57static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
  58
  59struct pci_pme_device {
  60	struct list_head list;
  61	struct pci_dev *dev;
  62};
  63
  64#define PME_TIMEOUT 1000 /* How long between PME checks */
  65
  66/*
  67 * Following exit from Conventional Reset, devices must be ready within 1 sec
  68 * (PCIe r6.0 sec 6.6.1).  A D3cold to D0 transition implies a Conventional
  69 * Reset (PCIe r6.0 sec 5.8).
  70 */
  71#define PCI_RESET_WAIT 1000 /* msec */
  72
  73/*
  74 * Devices may extend the 1 sec period through Request Retry Status
  75 * completions (PCIe r6.0 sec 2.3.1).  The spec does not provide an upper
  76 * limit, but 60 sec ought to be enough for any device to become
  77 * responsive.
  78 */
  79#define PCIE_RESET_READY_POLL_MS 60000 /* msec */
  80
  81static void pci_dev_d3_sleep(struct pci_dev *dev)
  82{
  83	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
  84	unsigned int upper;
  85
  86	if (delay_ms) {
  87		/* Use a 20% upper bound, 1ms minimum */
  88		upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
  89		usleep_range(delay_ms * USEC_PER_MSEC,
  90			     (delay_ms + upper) * USEC_PER_MSEC);
  91	}
  92}
  93
  94bool pci_reset_supported(struct pci_dev *dev)
  95{
  96	return dev->reset_methods[0] != 0;
  97}
  98
  99#ifdef CONFIG_PCI_DOMAINS
 100int pci_domains_supported = 1;
 101#endif
 102
 103#define DEFAULT_CARDBUS_IO_SIZE		(256)
 104#define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
 105/* pci=cbmemsize=nnM,cbiosize=nn can override this */
 106unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
 107unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
 108
 109#define DEFAULT_HOTPLUG_IO_SIZE		(256)
 110#define DEFAULT_HOTPLUG_MMIO_SIZE	(2*1024*1024)
 111#define DEFAULT_HOTPLUG_MMIO_PREF_SIZE	(2*1024*1024)
 112/* hpiosize=nn can override this */
 113unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
 114/*
 115 * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
 116 * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
 117 * pci=hpmemsize=nnM overrides both
 118 */
 119unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
 120unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
 121
 122#define DEFAULT_HOTPLUG_BUS_SIZE	1
 123unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
 124
 125
 126/* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
 127#ifdef CONFIG_PCIE_BUS_TUNE_OFF
 128enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
 129#elif defined CONFIG_PCIE_BUS_SAFE
 130enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
 131#elif defined CONFIG_PCIE_BUS_PERFORMANCE
 132enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
 133#elif defined CONFIG_PCIE_BUS_PEER2PEER
 134enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
 135#else
 136enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
 137#endif
 138
 139/*
 140 * The default CLS is used if arch didn't set CLS explicitly and not
 141 * all pci devices agree on the same value.  Arch can override either
 142 * the dfl or actual value as it sees fit.  Don't forget this is
 143 * measured in 32-bit words, not bytes.
 144 */
 145u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
 146u8 pci_cache_line_size;
 147
 148/*
 149 * If we set up a device for bus mastering, we need to check the latency
 150 * timer as certain BIOSes forget to set it properly.
 151 */
 152unsigned int pcibios_max_latency = 255;
 153
 154/* If set, the PCIe ARI capability will not be used. */
 155static bool pcie_ari_disabled;
 156
 157/* If set, the PCIe ATS capability will not be used. */
 158static bool pcie_ats_disabled;
 159
 160/* If set, the PCI config space of each device is printed during boot. */
 161bool pci_early_dump;
 162
 163bool pci_ats_disabled(void)
 164{
 165	return pcie_ats_disabled;
 166}
 167EXPORT_SYMBOL_GPL(pci_ats_disabled);
 168
 169/* Disable bridge_d3 for all PCIe ports */
 170static bool pci_bridge_d3_disable;
 171/* Force bridge_d3 for all PCIe ports */
 172static bool pci_bridge_d3_force;
 173
 174static int __init pcie_port_pm_setup(char *str)
 175{
 176	if (!strcmp(str, "off"))
 177		pci_bridge_d3_disable = true;
 178	else if (!strcmp(str, "force"))
 179		pci_bridge_d3_force = true;
 180	return 1;
 181}
 182__setup("pcie_port_pm=", pcie_port_pm_setup);
 183
 
 
 
 184/**
 185 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
 186 * @bus: pointer to PCI bus structure to search
 187 *
 188 * Given a PCI bus, returns the highest PCI bus number present in the set
 189 * including the given PCI bus and its list of child PCI buses.
 190 */
 191unsigned char pci_bus_max_busnr(struct pci_bus *bus)
 192{
 193	struct pci_bus *tmp;
 194	unsigned char max, n;
 195
 196	max = bus->busn_res.end;
 197	list_for_each_entry(tmp, &bus->children, node) {
 198		n = pci_bus_max_busnr(tmp);
 199		if (n > max)
 200			max = n;
 201	}
 202	return max;
 203}
 204EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
 205
 206/**
 207 * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
 208 * @pdev: the PCI device
 209 *
 210 * Returns error bits set in PCI_STATUS and clears them.
 211 */
 212int pci_status_get_and_clear_errors(struct pci_dev *pdev)
 213{
 214	u16 status;
 215	int ret;
 216
 217	ret = pci_read_config_word(pdev, PCI_STATUS, &status);
 218	if (ret != PCIBIOS_SUCCESSFUL)
 219		return -EIO;
 220
 221	status &= PCI_STATUS_ERROR_BITS;
 222	if (status)
 223		pci_write_config_word(pdev, PCI_STATUS, status);
 224
 225	return status;
 226}
 227EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
 228
 229#ifdef CONFIG_HAS_IOMEM
 230static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar,
 231					    bool write_combine)
 232{
 233	struct resource *res = &pdev->resource[bar];
 234	resource_size_t start = res->start;
 235	resource_size_t size = resource_size(res);
 236
 237	/*
 238	 * Make sure the BAR is actually a memory resource, not an IO resource
 239	 */
 240	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
 241		pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
 242		return NULL;
 243	}
 244
 245	if (write_combine)
 246		return ioremap_wc(start, size);
 247
 248	return ioremap(start, size);
 249}
 250
 251void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
 252{
 253	return __pci_ioremap_resource(pdev, bar, false);
 254}
 255EXPORT_SYMBOL_GPL(pci_ioremap_bar);
 256
 257void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
 258{
 259	return __pci_ioremap_resource(pdev, bar, true);
 
 
 
 
 
 
 
 
 260}
 261EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
 262#endif
 263
 264/**
 265 * pci_dev_str_match_path - test if a path string matches a device
 266 * @dev: the PCI device to test
 267 * @path: string to match the device against
 268 * @endptr: pointer to the string after the match
 269 *
 270 * Test if a string (typically from a kernel parameter) formatted as a
 271 * path of device/function addresses matches a PCI device. The string must
 272 * be of the form:
 273 *
 274 *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
 275 *
 276 * A path for a device can be obtained using 'lspci -t'.  Using a path
 277 * is more robust against bus renumbering than using only a single bus,
 278 * device and function address.
 279 *
 280 * Returns 1 if the string matches the device, 0 if it does not and
 281 * a negative error code if it fails to parse the string.
 282 */
 283static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
 284				  const char **endptr)
 285{
 286	int ret;
 287	unsigned int seg, bus, slot, func;
 288	char *wpath, *p;
 289	char end;
 290
 291	*endptr = strchrnul(path, ';');
 292
 293	wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC);
 294	if (!wpath)
 295		return -ENOMEM;
 296
 297	while (1) {
 298		p = strrchr(wpath, '/');
 299		if (!p)
 300			break;
 301		ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
 302		if (ret != 2) {
 303			ret = -EINVAL;
 304			goto free_and_exit;
 305		}
 306
 307		if (dev->devfn != PCI_DEVFN(slot, func)) {
 308			ret = 0;
 309			goto free_and_exit;
 310		}
 311
 312		/*
 313		 * Note: we don't need to get a reference to the upstream
 314		 * bridge because we hold a reference to the top level
 315		 * device which should hold a reference to the bridge,
 316		 * and so on.
 317		 */
 318		dev = pci_upstream_bridge(dev);
 319		if (!dev) {
 320			ret = 0;
 321			goto free_and_exit;
 322		}
 323
 324		*p = 0;
 325	}
 326
 327	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
 328		     &func, &end);
 329	if (ret != 4) {
 330		seg = 0;
 331		ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
 332		if (ret != 3) {
 333			ret = -EINVAL;
 334			goto free_and_exit;
 335		}
 336	}
 337
 338	ret = (seg == pci_domain_nr(dev->bus) &&
 339	       bus == dev->bus->number &&
 340	       dev->devfn == PCI_DEVFN(slot, func));
 341
 342free_and_exit:
 343	kfree(wpath);
 344	return ret;
 345}
 346
 347/**
 348 * pci_dev_str_match - test if a string matches a device
 349 * @dev: the PCI device to test
 350 * @p: string to match the device against
 351 * @endptr: pointer to the string after the match
 352 *
 353 * Test if a string (typically from a kernel parameter) matches a specified
 354 * PCI device. The string may be of one of the following formats:
 355 *
 356 *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
 357 *   pci:<vendor>:<device>[:<subvendor>:<subdevice>]
 358 *
 359 * The first format specifies a PCI bus/device/function address which
 360 * may change if new hardware is inserted, if motherboard firmware changes,
 361 * or due to changes caused in kernel parameters. If the domain is
 362 * left unspecified, it is taken to be 0.  In order to be robust against
 363 * bus renumbering issues, a path of PCI device/function numbers may be used
 364 * to address the specific device.  The path for a device can be determined
 365 * through the use of 'lspci -t'.
 366 *
 367 * The second format matches devices using IDs in the configuration
 368 * space which may match multiple devices in the system. A value of 0
 369 * for any field will match all devices. (Note: this differs from
 370 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for
 371 * legacy reasons and convenience so users don't have to specify
 372 * FFFFFFFFs on the command line.)
 373 *
 374 * Returns 1 if the string matches the device, 0 if it does not and
 375 * a negative error code if the string cannot be parsed.
 376 */
 377static int pci_dev_str_match(struct pci_dev *dev, const char *p,
 378			     const char **endptr)
 379{
 380	int ret;
 381	int count;
 382	unsigned short vendor, device, subsystem_vendor, subsystem_device;
 383
 384	if (strncmp(p, "pci:", 4) == 0) {
 385		/* PCI vendor/device (subvendor/subdevice) IDs are specified */
 386		p += 4;
 387		ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
 388			     &subsystem_vendor, &subsystem_device, &count);
 389		if (ret != 4) {
 390			ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
 391			if (ret != 2)
 392				return -EINVAL;
 393
 394			subsystem_vendor = 0;
 395			subsystem_device = 0;
 396		}
 397
 398		p += count;
 399
 400		if ((!vendor || vendor == dev->vendor) &&
 401		    (!device || device == dev->device) &&
 402		    (!subsystem_vendor ||
 403			    subsystem_vendor == dev->subsystem_vendor) &&
 404		    (!subsystem_device ||
 405			    subsystem_device == dev->subsystem_device))
 406			goto found;
 407	} else {
 408		/*
 409		 * PCI Bus, Device, Function IDs are specified
 410		 * (optionally, may include a path of devfns following it)
 411		 */
 412		ret = pci_dev_str_match_path(dev, p, &p);
 413		if (ret < 0)
 414			return ret;
 415		else if (ret)
 416			goto found;
 417	}
 418
 419	*endptr = p;
 420	return 0;
 421
 422found:
 423	*endptr = p;
 424	return 1;
 425}
 426
 427static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
 428				  u8 pos, int cap, int *ttl)
 429{
 430	u8 id;
 431	u16 ent;
 432
 433	pci_bus_read_config_byte(bus, devfn, pos, &pos);
 434
 435	while ((*ttl)--) {
 436		if (pos < 0x40)
 437			break;
 438		pos &= ~3;
 439		pci_bus_read_config_word(bus, devfn, pos, &ent);
 440
 441		id = ent & 0xff;
 442		if (id == 0xff)
 443			break;
 444		if (id == cap)
 445			return pos;
 446		pos = (ent >> 8);
 447	}
 448	return 0;
 449}
 450
 451static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
 452			      u8 pos, int cap)
 453{
 454	int ttl = PCI_FIND_CAP_TTL;
 455
 456	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
 457}
 458
 459u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
 460{
 461	return __pci_find_next_cap(dev->bus, dev->devfn,
 462				   pos + PCI_CAP_LIST_NEXT, cap);
 463}
 464EXPORT_SYMBOL_GPL(pci_find_next_capability);
 465
 466static u8 __pci_bus_find_cap_start(struct pci_bus *bus,
 467				    unsigned int devfn, u8 hdr_type)
 468{
 469	u16 status;
 470
 471	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
 472	if (!(status & PCI_STATUS_CAP_LIST))
 473		return 0;
 474
 475	switch (hdr_type) {
 476	case PCI_HEADER_TYPE_NORMAL:
 477	case PCI_HEADER_TYPE_BRIDGE:
 478		return PCI_CAPABILITY_LIST;
 479	case PCI_HEADER_TYPE_CARDBUS:
 480		return PCI_CB_CAPABILITY_LIST;
 481	}
 482
 483	return 0;
 484}
 485
 486/**
 487 * pci_find_capability - query for devices' capabilities
 488 * @dev: PCI device to query
 489 * @cap: capability code
 490 *
 491 * Tell if a device supports a given PCI capability.
 492 * Returns the address of the requested capability structure within the
 493 * device's PCI configuration space or 0 in case the device does not
 494 * support it.  Possible values for @cap include:
 495 *
 496 *  %PCI_CAP_ID_PM           Power Management
 497 *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
 498 *  %PCI_CAP_ID_VPD          Vital Product Data
 499 *  %PCI_CAP_ID_SLOTID       Slot Identification
 500 *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
 501 *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
 502 *  %PCI_CAP_ID_PCIX         PCI-X
 503 *  %PCI_CAP_ID_EXP          PCI Express
 504 */
 505u8 pci_find_capability(struct pci_dev *dev, int cap)
 506{
 507	u8 pos;
 508
 509	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
 510	if (pos)
 511		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
 512
 513	return pos;
 514}
 515EXPORT_SYMBOL(pci_find_capability);
 516
 517/**
 518 * pci_bus_find_capability - query for devices' capabilities
 519 * @bus: the PCI bus to query
 520 * @devfn: PCI device to query
 521 * @cap: capability code
 522 *
 523 * Like pci_find_capability() but works for PCI devices that do not have a
 524 * pci_dev structure set up yet.
 525 *
 526 * Returns the address of the requested capability structure within the
 527 * device's PCI configuration space or 0 in case the device does not
 528 * support it.
 529 */
 530u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
 531{
 532	u8 hdr_type, pos;
 533
 534	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
 535
 536	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & PCI_HEADER_TYPE_MASK);
 537	if (pos)
 538		pos = __pci_find_next_cap(bus, devfn, pos, cap);
 539
 540	return pos;
 541}
 542EXPORT_SYMBOL(pci_bus_find_capability);
 543
 544/**
 545 * pci_find_next_ext_capability - Find an extended capability
 546 * @dev: PCI device to query
 547 * @start: address at which to start looking (0 to start at beginning of list)
 548 * @cap: capability code
 549 *
 550 * Returns the address of the next matching extended capability structure
 551 * within the device's PCI configuration space or 0 if the device does
 552 * not support it.  Some capabilities can occur several times, e.g., the
 553 * vendor-specific capability, and this provides a way to find them all.
 554 */
 555u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap)
 556{
 557	u32 header;
 558	int ttl;
 559	u16 pos = PCI_CFG_SPACE_SIZE;
 560
 561	/* minimum 8 bytes per capability */
 562	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
 563
 564	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
 565		return 0;
 566
 567	if (start)
 568		pos = start;
 569
 570	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
 571		return 0;
 572
 573	/*
 574	 * If we have no capabilities, this is indicated by cap ID,
 575	 * cap version and next pointer all being 0.
 576	 */
 577	if (header == 0)
 578		return 0;
 579
 580	while (ttl-- > 0) {
 581		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
 582			return pos;
 583
 584		pos = PCI_EXT_CAP_NEXT(header);
 585		if (pos < PCI_CFG_SPACE_SIZE)
 586			break;
 587
 588		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
 589			break;
 590	}
 591
 592	return 0;
 593}
 594EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
 595
 596/**
 597 * pci_find_ext_capability - Find an extended capability
 598 * @dev: PCI device to query
 599 * @cap: capability code
 600 *
 601 * Returns the address of the requested extended capability structure
 602 * within the device's PCI configuration space or 0 if the device does
 603 * not support it.  Possible values for @cap include:
 604 *
 605 *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
 606 *  %PCI_EXT_CAP_ID_VC		Virtual Channel
 607 *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
 608 *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
 609 */
 610u16 pci_find_ext_capability(struct pci_dev *dev, int cap)
 611{
 612	return pci_find_next_ext_capability(dev, 0, cap);
 613}
 614EXPORT_SYMBOL_GPL(pci_find_ext_capability);
 615
 616/**
 617 * pci_get_dsn - Read and return the 8-byte Device Serial Number
 618 * @dev: PCI device to query
 619 *
 620 * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
 621 * Number.
 622 *
 623 * Returns the DSN, or zero if the capability does not exist.
 624 */
 625u64 pci_get_dsn(struct pci_dev *dev)
 626{
 627	u32 dword;
 628	u64 dsn;
 629	int pos;
 630
 631	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
 632	if (!pos)
 633		return 0;
 634
 635	/*
 636	 * The Device Serial Number is two dwords offset 4 bytes from the
 637	 * capability position. The specification says that the first dword is
 638	 * the lower half, and the second dword is the upper half.
 639	 */
 640	pos += 4;
 641	pci_read_config_dword(dev, pos, &dword);
 642	dsn = (u64)dword;
 643	pci_read_config_dword(dev, pos + 4, &dword);
 644	dsn |= ((u64)dword) << 32;
 645
 646	return dsn;
 647}
 648EXPORT_SYMBOL_GPL(pci_get_dsn);
 649
 650static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap)
 651{
 652	int rc, ttl = PCI_FIND_CAP_TTL;
 653	u8 cap, mask;
 654
 655	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
 656		mask = HT_3BIT_CAP_MASK;
 657	else
 658		mask = HT_5BIT_CAP_MASK;
 659
 660	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
 661				      PCI_CAP_ID_HT, &ttl);
 662	while (pos) {
 663		rc = pci_read_config_byte(dev, pos + 3, &cap);
 664		if (rc != PCIBIOS_SUCCESSFUL)
 665			return 0;
 666
 667		if ((cap & mask) == ht_cap)
 668			return pos;
 669
 670		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
 671					      pos + PCI_CAP_LIST_NEXT,
 672					      PCI_CAP_ID_HT, &ttl);
 673	}
 674
 675	return 0;
 676}
 677
 678/**
 679 * pci_find_next_ht_capability - query a device's HyperTransport capabilities
 680 * @dev: PCI device to query
 681 * @pos: Position from which to continue searching
 682 * @ht_cap: HyperTransport capability code
 683 *
 684 * To be used in conjunction with pci_find_ht_capability() to search for
 685 * all capabilities matching @ht_cap. @pos should always be a value returned
 686 * from pci_find_ht_capability().
 687 *
 688 * NB. To be 100% safe against broken PCI devices, the caller should take
 689 * steps to avoid an infinite loop.
 690 */
 691u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap)
 692{
 693	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
 694}
 695EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
 696
 697/**
 698 * pci_find_ht_capability - query a device's HyperTransport capabilities
 699 * @dev: PCI device to query
 700 * @ht_cap: HyperTransport capability code
 701 *
 702 * Tell if a device supports a given HyperTransport capability.
 703 * Returns an address within the device's PCI configuration space
 704 * or 0 in case the device does not support the request capability.
 705 * The address points to the PCI capability, of type PCI_CAP_ID_HT,
 706 * which has a HyperTransport capability matching @ht_cap.
 707 */
 708u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
 709{
 710	u8 pos;
 711
 712	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
 713	if (pos)
 714		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
 715
 716	return pos;
 717}
 718EXPORT_SYMBOL_GPL(pci_find_ht_capability);
 719
 720/**
 721 * pci_find_vsec_capability - Find a vendor-specific extended capability
 722 * @dev: PCI device to query
 723 * @vendor: Vendor ID for which capability is defined
 724 * @cap: Vendor-specific capability ID
 725 *
 726 * If @dev has Vendor ID @vendor, search for a VSEC capability with
 727 * VSEC ID @cap. If found, return the capability offset in
 728 * config space; otherwise return 0.
 729 */
 730u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
 731{
 732	u16 vsec = 0;
 733	u32 header;
 734	int ret;
 735
 736	if (vendor != dev->vendor)
 737		return 0;
 738
 739	while ((vsec = pci_find_next_ext_capability(dev, vsec,
 740						     PCI_EXT_CAP_ID_VNDR))) {
 741		ret = pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header);
 742		if (ret != PCIBIOS_SUCCESSFUL)
 743			continue;
 744
 745		if (PCI_VNDR_HEADER_ID(header) == cap)
 746			return vsec;
 747	}
 748
 749	return 0;
 750}
 751EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
 752
 753/**
 754 * pci_find_dvsec_capability - Find DVSEC for vendor
 755 * @dev: PCI device to query
 756 * @vendor: Vendor ID to match for the DVSEC
 757 * @dvsec: Designated Vendor-specific capability ID
 758 *
 759 * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
 760 * offset in config space; otherwise return 0.
 761 */
 762u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
 763{
 764	int pos;
 765
 766	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
 767	if (!pos)
 768		return 0;
 769
 770	while (pos) {
 771		u16 v, id;
 772
 773		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v);
 774		pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id);
 775		if (vendor == v && dvsec == id)
 776			return pos;
 777
 778		pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
 779	}
 780
 781	return 0;
 782}
 783EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
 784
 785/**
 786 * pci_find_parent_resource - return resource region of parent bus of given
 787 *			      region
 788 * @dev: PCI device structure contains resources to be searched
 789 * @res: child resource record for which parent is sought
 790 *
 791 * For given resource region of given device, return the resource region of
 792 * parent bus the given region is contained in.
 793 */
 794struct resource *pci_find_parent_resource(const struct pci_dev *dev,
 795					  struct resource *res)
 796{
 797	const struct pci_bus *bus = dev->bus;
 798	struct resource *r;
 
 799
 800	pci_bus_for_each_resource(bus, r) {
 801		if (!r)
 802			continue;
 803		if (resource_contains(r, res)) {
 804
 805			/*
 806			 * If the window is prefetchable but the BAR is
 807			 * not, the allocator made a mistake.
 808			 */
 809			if (r->flags & IORESOURCE_PREFETCH &&
 810			    !(res->flags & IORESOURCE_PREFETCH))
 811				return NULL;
 812
 813			/*
 814			 * If we're below a transparent bridge, there may
 815			 * be both a positively-decoded aperture and a
 816			 * subtractively-decoded region that contain the BAR.
 817			 * We want the positively-decoded one, so this depends
 818			 * on pci_bus_for_each_resource() giving us those
 819			 * first.
 820			 */
 821			return r;
 822		}
 823	}
 824	return NULL;
 825}
 826EXPORT_SYMBOL(pci_find_parent_resource);
 827
 828/**
 829 * pci_find_resource - Return matching PCI device resource
 830 * @dev: PCI device to query
 831 * @res: Resource to look for
 832 *
 833 * Goes over standard PCI resources (BARs) and checks if the given resource
 834 * is partially or fully contained in any of them. In that case the
 835 * matching resource is returned, %NULL otherwise.
 836 */
 837struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
 838{
 839	int i;
 840
 841	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 842		struct resource *r = &dev->resource[i];
 843
 844		if (r->start && resource_contains(r, res))
 845			return r;
 846	}
 847
 848	return NULL;
 849}
 850EXPORT_SYMBOL(pci_find_resource);
 851
 852/**
 853 * pci_resource_name - Return the name of the PCI resource
 854 * @dev: PCI device to query
 855 * @i: index of the resource
 856 *
 857 * Return the standard PCI resource (BAR) name according to their index.
 858 */
 859const char *pci_resource_name(struct pci_dev *dev, unsigned int i)
 860{
 861	static const char * const bar_name[] = {
 862		"BAR 0",
 863		"BAR 1",
 864		"BAR 2",
 865		"BAR 3",
 866		"BAR 4",
 867		"BAR 5",
 868		"ROM",
 869#ifdef CONFIG_PCI_IOV
 870		"VF BAR 0",
 871		"VF BAR 1",
 872		"VF BAR 2",
 873		"VF BAR 3",
 874		"VF BAR 4",
 875		"VF BAR 5",
 876#endif
 877		"bridge window",	/* "io" included in %pR */
 878		"bridge window",	/* "mem" included in %pR */
 879		"bridge window",	/* "mem pref" included in %pR */
 880	};
 881	static const char * const cardbus_name[] = {
 882		"BAR 1",
 883		"unknown",
 884		"unknown",
 885		"unknown",
 886		"unknown",
 887		"unknown",
 888#ifdef CONFIG_PCI_IOV
 889		"unknown",
 890		"unknown",
 891		"unknown",
 892		"unknown",
 893		"unknown",
 894		"unknown",
 895#endif
 896		"CardBus bridge window 0",	/* I/O */
 897		"CardBus bridge window 1",	/* I/O */
 898		"CardBus bridge window 0",	/* mem */
 899		"CardBus bridge window 1",	/* mem */
 900	};
 901
 902	if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS &&
 903	    i < ARRAY_SIZE(cardbus_name))
 904		return cardbus_name[i];
 905
 906	if (i < ARRAY_SIZE(bar_name))
 907		return bar_name[i];
 908
 909	return "unknown";
 910}
 911
 912/**
 913 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
 914 * @dev: the PCI device to operate on
 915 * @pos: config space offset of status word
 916 * @mask: mask of bit(s) to care about in status word
 917 *
 918 * Return 1 when mask bit(s) in status word clear, 0 otherwise.
 919 */
 920int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
 921{
 922	int i;
 923
 924	/* Wait for Transaction Pending bit clean */
 925	for (i = 0; i < 4; i++) {
 926		u16 status;
 927		if (i)
 928			msleep((1 << (i - 1)) * 100);
 929
 930		pci_read_config_word(dev, pos, &status);
 931		if (!(status & mask))
 932			return 1;
 933	}
 934
 935	return 0;
 936}
 937
 938static int pci_acs_enable;
 939
 940/**
 941 * pci_request_acs - ask for ACS to be enabled if supported
 942 */
 943void pci_request_acs(void)
 944{
 945	pci_acs_enable = 1;
 946}
 947
 948static const char *disable_acs_redir_param;
 949
 950/**
 951 * pci_disable_acs_redir - disable ACS redirect capabilities
 952 * @dev: the PCI device
 953 *
 954 * For only devices specified in the disable_acs_redir parameter.
 955 */
 956static void pci_disable_acs_redir(struct pci_dev *dev)
 957{
 958	int ret = 0;
 959	const char *p;
 960	int pos;
 961	u16 ctrl;
 962
 963	if (!disable_acs_redir_param)
 964		return;
 965
 966	p = disable_acs_redir_param;
 967	while (*p) {
 968		ret = pci_dev_str_match(dev, p, &p);
 969		if (ret < 0) {
 970			pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n",
 971				     disable_acs_redir_param);
 972
 973			break;
 974		} else if (ret == 1) {
 975			/* Found a match */
 976			break;
 977		}
 978
 979		if (*p != ';' && *p != ',') {
 980			/* End of param or invalid format */
 981			break;
 982		}
 983		p++;
 984	}
 985
 986	if (ret != 1)
 987		return;
 988
 989	if (!pci_dev_specific_disable_acs_redir(dev))
 990		return;
 991
 992	pos = dev->acs_cap;
 993	if (!pos) {
 994		pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n");
 995		return;
 996	}
 997
 998	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
 999
1000	/* P2P Request & Completion Redirect */
1001	ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC);
1002
1003	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
1004
1005	pci_info(dev, "disabled ACS redirect\n");
1006}
1007
1008/**
1009 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
1010 * @dev: the PCI device
1011 */
1012static void pci_std_enable_acs(struct pci_dev *dev)
1013{
1014	int pos;
1015	u16 cap;
1016	u16 ctrl;
1017
1018	pos = dev->acs_cap;
1019	if (!pos)
1020		return;
1021
1022	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
1023	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
1024
1025	/* Source Validation */
1026	ctrl |= (cap & PCI_ACS_SV);
1027
1028	/* P2P Request Redirect */
1029	ctrl |= (cap & PCI_ACS_RR);
1030
1031	/* P2P Completion Redirect */
1032	ctrl |= (cap & PCI_ACS_CR);
1033
1034	/* Upstream Forwarding */
1035	ctrl |= (cap & PCI_ACS_UF);
1036
1037	/* Enable Translation Blocking for external devices and noats */
1038	if (pci_ats_disabled() || dev->external_facing || dev->untrusted)
1039		ctrl |= (cap & PCI_ACS_TB);
1040
1041	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
1042}
1043
1044/**
1045 * pci_enable_acs - enable ACS if hardware support it
1046 * @dev: the PCI device
1047 */
1048static void pci_enable_acs(struct pci_dev *dev)
1049{
1050	if (!pci_acs_enable)
1051		goto disable_acs_redir;
1052
1053	if (!pci_dev_specific_enable_acs(dev))
1054		goto disable_acs_redir;
1055
1056	pci_std_enable_acs(dev);
1057
1058disable_acs_redir:
1059	/*
1060	 * Note: pci_disable_acs_redir() must be called even if ACS was not
1061	 * enabled by the kernel because it may have been enabled by
1062	 * platform firmware.  So if we are told to disable it, we should
1063	 * always disable it after setting the kernel's default
1064	 * preferences.
1065	 */
1066	pci_disable_acs_redir(dev);
1067}
1068
1069/**
1070 * pcie_read_tlp_log - read TLP Header Log
1071 * @dev: PCIe device
1072 * @where: PCI Config offset of TLP Header Log
1073 * @tlp_log: TLP Log structure to fill
1074 *
1075 * Fill @tlp_log from TLP Header Log registers, e.g., AER or DPC.
1076 *
1077 * Return: 0 on success and filled TLP Log structure, <0 on error.
1078 */
1079int pcie_read_tlp_log(struct pci_dev *dev, int where,
1080		      struct pcie_tlp_log *tlp_log)
1081{
1082	int i, ret;
1083
1084	memset(tlp_log, 0, sizeof(*tlp_log));
1085
1086	for (i = 0; i < 4; i++) {
1087		ret = pci_read_config_dword(dev, where + i * 4,
1088					    &tlp_log->dw[i]);
1089		if (ret)
1090			return pcibios_err_to_errno(ret);
1091	}
1092
1093	return 0;
1094}
1095EXPORT_SYMBOL_GPL(pcie_read_tlp_log);
1096
1097/**
1098 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
1099 * @dev: PCI device to have its BARs restored
1100 *
1101 * Restore the BAR values for a given device, so as to make it
1102 * accessible by its driver.
1103 */
1104static void pci_restore_bars(struct pci_dev *dev)
1105{
1106	int i;
1107
1108	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
1109		pci_update_resource(dev, i);
1110}
1111
1112static inline bool platform_pci_power_manageable(struct pci_dev *dev)
 
 
1113{
1114	if (pci_use_mid_pm())
1115		return true;
 
 
 
 
1116
1117	return acpi_pci_power_manageable(dev);
 
 
1118}
1119
1120static inline int platform_pci_set_power_state(struct pci_dev *dev,
1121					       pci_power_t t)
1122{
1123	if (pci_use_mid_pm())
1124		return mid_pci_set_power_state(dev, t);
1125
1126	return acpi_pci_set_power_state(dev, t);
1127}
1128
1129static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
1130{
1131	if (pci_use_mid_pm())
1132		return mid_pci_get_power_state(dev);
1133
1134	return acpi_pci_get_power_state(dev);
1135}
1136
1137static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
1138{
1139	if (!pci_use_mid_pm())
1140		acpi_pci_refresh_power_state(dev);
1141}
1142
1143static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
1144{
1145	if (pci_use_mid_pm())
1146		return PCI_POWER_ERROR;
1147
1148	return acpi_pci_choose_state(dev);
1149}
1150
1151static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
1152{
1153	if (pci_use_mid_pm())
1154		return PCI_POWER_ERROR;
1155
1156	return acpi_pci_wakeup(dev, enable);
1157}
1158
1159static inline bool platform_pci_need_resume(struct pci_dev *dev)
1160{
1161	if (pci_use_mid_pm())
1162		return false;
1163
1164	return acpi_pci_need_resume(dev);
1165}
1166
1167static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
1168{
1169	if (pci_use_mid_pm())
1170		return false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1171
1172	return acpi_pci_bridge_d3(dev);
 
 
 
1173}
1174
1175/**
1176 * pci_update_current_state - Read power state of given device and cache it
1177 * @dev: PCI device to handle.
1178 * @state: State to cache in case the device doesn't have the PM capability
1179 *
1180 * The power state is read from the PMCSR register, which however is
1181 * inaccessible in D3cold.  The platform firmware is therefore queried first
1182 * to detect accessibility of the register.  In case the platform firmware
1183 * reports an incorrect state or the device isn't power manageable by the
1184 * platform at all, we try to detect D3cold by testing accessibility of the
1185 * vendor ID in config space.
1186 */
1187void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
1188{
1189	if (platform_pci_get_power_state(dev) == PCI_D3cold) {
 
1190		dev->current_state = PCI_D3cold;
1191	} else if (dev->pm_cap) {
1192		u16 pmcsr;
1193
1194		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1195		if (PCI_POSSIBLE_ERROR(pmcsr)) {
1196			dev->current_state = PCI_D3cold;
1197			return;
1198		}
1199		dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1200	} else {
1201		dev->current_state = state;
1202	}
1203}
1204
1205/**
1206 * pci_refresh_power_state - Refresh the given device's power state data
1207 * @dev: Target PCI device.
1208 *
1209 * Ask the platform to refresh the devices power state information and invoke
1210 * pci_update_current_state() to update its current PCI power state.
1211 */
1212void pci_refresh_power_state(struct pci_dev *dev)
1213{
1214	platform_pci_refresh_power_state(dev);
 
 
1215	pci_update_current_state(dev, dev->current_state);
1216}
1217
1218/**
1219 * pci_platform_power_transition - Use platform to change device power state
1220 * @dev: PCI device to handle.
1221 * @state: State to put the device into.
1222 */
1223int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
1224{
1225	int error;
1226
1227	error = platform_pci_set_power_state(dev, state);
1228	if (!error)
1229		pci_update_current_state(dev, state);
1230	else if (!dev->pm_cap) /* Fall back to PCI_D0 */
 
 
 
 
1231		dev->current_state = PCI_D0;
1232
1233	return error;
1234}
1235EXPORT_SYMBOL_GPL(pci_platform_power_transition);
1236
1237static int pci_resume_one(struct pci_dev *pci_dev, void *ign)
1238{
1239	pm_request_resume(&pci_dev->dev);
1240	return 0;
1241}
1242
1243/**
1244 * pci_resume_bus - Walk given bus and runtime resume devices on it
1245 * @bus: Top bus of the subtree to walk.
1246 */
1247void pci_resume_bus(struct pci_bus *bus)
1248{
1249	if (bus)
1250		pci_walk_bus(bus, pci_resume_one, NULL);
1251}
1252
1253static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
1254{
1255	int delay = 1;
1256	bool retrain = false;
1257	struct pci_dev *bridge;
1258
1259	if (pci_is_pcie(dev)) {
1260		bridge = pci_upstream_bridge(dev);
1261		if (bridge)
1262			retrain = true;
1263	}
1264
1265	/*
1266	 * After reset, the device should not silently discard config
1267	 * requests, but it may still indicate that it needs more time by
1268	 * responding to them with CRS completions.  The Root Port will
1269	 * generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete
1270	 * the read (except when CRS SV is enabled and the read was for the
1271	 * Vendor ID; in that case it synthesizes 0x0001 data).
1272	 *
1273	 * Wait for the device to return a non-CRS completion.  Read the
1274	 * Command register instead of Vendor ID so we don't have to
1275	 * contend with the CRS SV value.
1276	 */
1277	for (;;) {
1278		u32 id;
1279
1280		pci_read_config_dword(dev, PCI_COMMAND, &id);
1281		if (!PCI_POSSIBLE_ERROR(id))
1282			break;
1283
1284		if (delay > timeout) {
1285			pci_warn(dev, "not ready %dms after %s; giving up\n",
1286				 delay - 1, reset_type);
1287			return -ENOTTY;
1288		}
1289
1290		if (delay > PCI_RESET_WAIT) {
1291			if (retrain) {
1292				retrain = false;
1293				if (pcie_failed_link_retrain(bridge)) {
1294					delay = 1;
1295					continue;
1296				}
1297			}
1298			pci_info(dev, "not ready %dms after %s; waiting\n",
1299				 delay - 1, reset_type);
1300		}
1301
1302		msleep(delay);
1303		delay *= 2;
 
1304	}
1305
1306	if (delay > PCI_RESET_WAIT)
1307		pci_info(dev, "ready %dms after %s\n", delay - 1,
1308			 reset_type);
1309	else
1310		pci_dbg(dev, "ready %dms after %s\n", delay - 1,
1311			reset_type);
1312
1313	return 0;
1314}
1315
1316/**
1317 * pci_power_up - Put the given device into D0
1318 * @dev: PCI device to power up
1319 *
1320 * On success, return 0 or 1, depending on whether or not it is necessary to
1321 * restore the device's BARs subsequently (1 is returned in that case).
1322 *
1323 * On failure, return a negative error code.  Always return failure if @dev
1324 * lacks a Power Management Capability, even if the platform was able to
1325 * put the device in D0 via non-PCI means.
1326 */
1327int pci_power_up(struct pci_dev *dev)
1328{
1329	bool need_restore;
1330	pci_power_t state;
1331	u16 pmcsr;
1332
1333	platform_pci_set_power_state(dev, PCI_D0);
1334
1335	if (!dev->pm_cap) {
1336		state = platform_pci_get_power_state(dev);
1337		if (state == PCI_UNKNOWN)
1338			dev->current_state = PCI_D0;
1339		else
1340			dev->current_state = state;
1341
1342		return -EIO;
1343	}
1344
1345	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1346	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1347		pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n",
1348			pci_power_name(dev->current_state));
1349		dev->current_state = PCI_D3cold;
1350		return -EIO;
1351	}
1352
1353	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1354
1355	need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) &&
1356			!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
1357
1358	if (state == PCI_D0)
1359		goto end;
1360
1361	/*
1362	 * Force the entire word to 0. This doesn't affect PME_Status, disables
1363	 * PME_En, and sets PowerState to 0.
 
1364	 */
1365	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0);
1366
1367	/* Mandatory transition delays; see PCI PM 1.2. */
1368	if (state == PCI_D3hot)
1369		pci_dev_d3_sleep(dev);
1370	else if (state == PCI_D2)
1371		udelay(PCI_PM_D2_DELAY);
1372
1373end:
1374	dev->current_state = PCI_D0;
1375	if (need_restore)
1376		return 1;
1377
1378	return 0;
1379}
1380
1381/**
1382 * pci_set_full_power_state - Put a PCI device into D0 and update its state
1383 * @dev: PCI device to power up
1384 * @locked: whether pci_bus_sem is held
1385 *
1386 * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register
1387 * to confirm the state change, restore its BARs if they might be lost and
1388 * reconfigure ASPM in accordance with the new power state.
1389 *
1390 * If pci_restore_state() is going to be called right after a power state change
1391 * to D0, it is more efficient to use pci_power_up() directly instead of this
1392 * function.
1393 */
1394static int pci_set_full_power_state(struct pci_dev *dev, bool locked)
1395{
1396	u16 pmcsr;
1397	int ret;
1398
1399	ret = pci_power_up(dev);
1400	if (ret < 0) {
1401		if (dev->current_state == PCI_D0)
1402			return 0;
1403
1404		return ret;
1405	}
1406
1407	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1408	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1409	if (dev->current_state != PCI_D0) {
1410		pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n",
1411				     pci_power_name(dev->current_state));
1412	} else if (ret > 0) {
1413		/*
1414		 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
1415		 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
1416		 * from D3hot to D0 _may_ perform an internal reset, thereby
1417		 * going to "D0 Uninitialized" rather than "D0 Initialized".
1418		 * For example, at least some versions of the 3c905B and the
1419		 * 3c556B exhibit this behaviour.
1420		 *
1421		 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
1422		 * devices in a D3hot state at boot.  Consequently, we need to
1423		 * restore at least the BARs so that the device will be
1424		 * accessible to its driver.
1425		 */
1426		pci_restore_bars(dev);
1427	}
1428
1429	if (dev->bus->self)
1430		pcie_aspm_pm_state_change(dev->bus->self, locked);
1431
1432	return 0;
1433}
1434
1435/**
1436 * __pci_dev_set_current_state - Set current state of a PCI device
1437 * @dev: Device to handle
1438 * @data: pointer to state to be set
1439 */
1440static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1441{
1442	pci_power_t state = *(pci_power_t *)data;
1443
1444	dev->current_state = state;
1445	return 0;
1446}
1447
1448/**
1449 * pci_bus_set_current_state - Walk given bus and set current state of devices
1450 * @bus: Top bus of the subtree to walk.
1451 * @state: state to be set
1452 */
1453void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1454{
1455	if (bus)
1456		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1457}
1458
1459static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state, bool locked)
1460{
1461	if (!bus)
1462		return;
1463
1464	if (locked)
1465		pci_walk_bus_locked(bus, __pci_dev_set_current_state, &state);
1466	else
1467		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1468}
1469
1470/**
1471 * pci_set_low_power_state - Put a PCI device into a low-power state.
1472 * @dev: PCI device to handle.
1473 * @state: PCI power state (D1, D2, D3hot) to put the device into.
1474 * @locked: whether pci_bus_sem is held
1475 *
1476 * Use the device's PCI_PM_CTRL register to put it into a low-power state.
 
1477 *
1478 * RETURN VALUE:
1479 * -EINVAL if the requested state is invalid.
1480 * -EIO if device does not support PCI PM or its PM capabilities register has a
1481 * wrong version, or device doesn't support the requested state.
 
1482 * 0 if device already is in the requested state.
 
1483 * 0 if device's power state has been successfully changed.
1484 */
1485static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1486{
1487	u16 pmcsr;
1488
1489	if (!dev->pm_cap)
1490		return -EIO;
1491
1492	/*
1493	 * Validate transition: We can enter D0 from any state, but if
1494	 * we're already in a low-power state, we can only go deeper.  E.g.,
1495	 * we can go from D1 to D3, but we can't go directly from D3 to D1;
1496	 * we'd have to go from D3 to D0, then to D1.
1497	 */
1498	if (dev->current_state <= PCI_D3cold && dev->current_state > state) {
1499		pci_dbg(dev, "Invalid power transition (from %s to %s)\n",
1500			pci_power_name(dev->current_state),
1501			pci_power_name(state));
1502		return -EINVAL;
1503	}
1504
1505	/* Check if this device supports the desired state */
1506	if ((state == PCI_D1 && !dev->d1_support)
1507	   || (state == PCI_D2 && !dev->d2_support))
1508		return -EIO;
1509
1510	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1511	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1512		pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n",
1513			pci_power_name(dev->current_state),
1514			pci_power_name(state));
1515		dev->current_state = PCI_D3cold;
1516		return -EIO;
1517	}
1518
1519	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1520	pmcsr |= state;
1521
1522	/* Enter specified state */
1523	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1524
1525	/* Mandatory power management transition delays; see PCI PM 1.2. */
1526	if (state == PCI_D3hot)
1527		pci_dev_d3_sleep(dev);
1528	else if (state == PCI_D2)
1529		udelay(PCI_PM_D2_DELAY);
1530
1531	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1532	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1533	if (dev->current_state != state)
1534		pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n",
1535				     pci_power_name(dev->current_state),
1536				     pci_power_name(state));
1537
1538	if (dev->bus->self)
1539		pcie_aspm_pm_state_change(dev->bus->self, locked);
1540
1541	return 0;
1542}
1543
1544static int __pci_set_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1545{
1546	int error;
1547
1548	/* Bound the state we're entering */
1549	if (state > PCI_D3cold)
1550		state = PCI_D3cold;
1551	else if (state < PCI_D0)
1552		state = PCI_D0;
1553	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1554
1555		/*
1556		 * If the device or the parent bridge do not support PCI
1557		 * PM, ignore the request if we're doing anything other
1558		 * than putting it into D0 (which would only happen on
1559		 * boot).
1560		 */
1561		return 0;
1562
1563	/* Check if we're already there */
1564	if (dev->current_state == state)
1565		return 0;
1566
1567	if (state == PCI_D0)
1568		return pci_set_full_power_state(dev, locked);
1569
1570	/*
1571	 * This device is quirked not to be put into D3, so don't put it in
1572	 * D3
1573	 */
1574	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1575		return 0;
1576
1577	if (state == PCI_D3cold) {
1578		/*
1579		 * To put the device in D3cold, put it into D3hot in the native
1580		 * way, then put it into D3cold using platform ops.
1581		 */
1582		error = pci_set_low_power_state(dev, PCI_D3hot, locked);
1583
1584		if (pci_platform_power_transition(dev, PCI_D3cold))
1585			return error;
1586
1587		/* Powering off a bridge may power off the whole hierarchy */
1588		if (dev->current_state == PCI_D3cold)
1589			__pci_bus_set_current_state(dev->subordinate, PCI_D3cold, locked);
1590	} else {
1591		error = pci_set_low_power_state(dev, state, locked);
1592
1593		if (pci_platform_power_transition(dev, state))
1594			return error;
1595	}
1596
1597	return 0;
1598}
 
1599
1600/**
1601 * pci_set_power_state - Set the power state of a PCI device
1602 * @dev: PCI device to handle.
1603 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1604 *
1605 * Transition a device to a new power state, using the platform firmware and/or
1606 * the device's PCI PM registers.
1607 *
1608 * RETURN VALUE:
1609 * -EINVAL if the requested state is invalid.
1610 * -EIO if device does not support PCI PM or its PM capabilities register has a
1611 * wrong version, or device doesn't support the requested state.
1612 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1613 * 0 if device already is in the requested state.
1614 * 0 if the transition is to D3 but D3 is not supported.
1615 * 0 if device's power state has been successfully changed.
1616 */
1617int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1618{
1619	return __pci_set_power_state(dev, state, false);
1620}
1621EXPORT_SYMBOL(pci_set_power_state);
1622
1623int pci_set_power_state_locked(struct pci_dev *dev, pci_power_t state)
1624{
1625	lockdep_assert_held(&pci_bus_sem);
1626
1627	return __pci_set_power_state(dev, state, true);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1628}
1629EXPORT_SYMBOL(pci_set_power_state_locked);
1630
1631#define PCI_EXP_SAVE_REGS	7
1632
1633static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1634						       u16 cap, bool extended)
1635{
1636	struct pci_cap_saved_state *tmp;
1637
1638	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1639		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1640			return tmp;
1641	}
1642	return NULL;
1643}
1644
1645struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1646{
1647	return _pci_find_saved_cap(dev, cap, false);
1648}
1649
1650struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1651{
1652	return _pci_find_saved_cap(dev, cap, true);
1653}
1654
1655static int pci_save_pcie_state(struct pci_dev *dev)
1656{
1657	int i = 0;
1658	struct pci_cap_saved_state *save_state;
1659	u16 *cap;
1660
1661	if (!pci_is_pcie(dev))
1662		return 0;
1663
1664	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1665	if (!save_state) {
1666		pci_err(dev, "buffer not found in %s\n", __func__);
1667		return -ENOMEM;
1668	}
1669
1670	cap = (u16 *)&save_state->cap.data[0];
1671	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
1672	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
1673	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
1674	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
1675	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
1676	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
1677	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1678
1679	pci_save_aspm_l1ss_state(dev);
1680	pci_save_ltr_state(dev);
1681
1682	return 0;
1683}
1684
1685static void pci_restore_pcie_state(struct pci_dev *dev)
1686{
1687	int i = 0;
1688	struct pci_cap_saved_state *save_state;
1689	u16 *cap;
1690
1691	/*
1692	 * Restore max latencies (in the LTR capability) before enabling
1693	 * LTR itself in PCI_EXP_DEVCTL2.
1694	 */
1695	pci_restore_ltr_state(dev);
1696	pci_restore_aspm_l1ss_state(dev);
1697
1698	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1699	if (!save_state)
1700		return;
1701
1702	/*
1703	 * Downstream ports reset the LTR enable bit when link goes down.
1704	 * Check and re-configure the bit here before restoring device.
1705	 * PCIe r5.0, sec 7.5.3.16.
1706	 */
1707	pci_bridge_reconfigure_ltr(dev);
1708
1709	cap = (u16 *)&save_state->cap.data[0];
1710	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
1711	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
1712	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
1713	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
1714	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
1715	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
1716	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1717}
1718
1719static int pci_save_pcix_state(struct pci_dev *dev)
1720{
1721	int pos;
1722	struct pci_cap_saved_state *save_state;
1723
1724	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1725	if (!pos)
1726		return 0;
1727
1728	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1729	if (!save_state) {
1730		pci_err(dev, "buffer not found in %s\n", __func__);
1731		return -ENOMEM;
1732	}
1733
1734	pci_read_config_word(dev, pos + PCI_X_CMD,
1735			     (u16 *)save_state->cap.data);
1736
1737	return 0;
1738}
1739
1740static void pci_restore_pcix_state(struct pci_dev *dev)
1741{
1742	int i = 0, pos;
1743	struct pci_cap_saved_state *save_state;
1744	u16 *cap;
1745
1746	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1747	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1748	if (!save_state || !pos)
1749		return;
1750	cap = (u16 *)&save_state->cap.data[0];
1751
1752	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
1753}
1754
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1755/**
1756 * pci_save_state - save the PCI configuration space of a device before
1757 *		    suspending
1758 * @dev: PCI device that we're dealing with
1759 */
1760int pci_save_state(struct pci_dev *dev)
1761{
1762	int i;
1763	/* XXX: 100% dword access ok here? */
1764	for (i = 0; i < 16; i++) {
1765		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1766		pci_dbg(dev, "save config %#04x: %#010x\n",
1767			i * 4, dev->saved_config_space[i]);
1768	}
1769	dev->state_saved = true;
1770
1771	i = pci_save_pcie_state(dev);
1772	if (i != 0)
1773		return i;
1774
1775	i = pci_save_pcix_state(dev);
1776	if (i != 0)
1777		return i;
1778
 
1779	pci_save_dpc_state(dev);
1780	pci_save_aer_state(dev);
1781	pci_save_ptm_state(dev);
1782	return pci_save_vc_state(dev);
1783}
1784EXPORT_SYMBOL(pci_save_state);
1785
1786static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1787				     u32 saved_val, int retry, bool force)
1788{
1789	u32 val;
1790
1791	pci_read_config_dword(pdev, offset, &val);
1792	if (!force && val == saved_val)
1793		return;
1794
1795	for (;;) {
1796		pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
1797			offset, val, saved_val);
1798		pci_write_config_dword(pdev, offset, saved_val);
1799		if (retry-- <= 0)
1800			return;
1801
1802		pci_read_config_dword(pdev, offset, &val);
1803		if (val == saved_val)
1804			return;
1805
1806		mdelay(1);
1807	}
1808}
1809
1810static void pci_restore_config_space_range(struct pci_dev *pdev,
1811					   int start, int end, int retry,
1812					   bool force)
1813{
1814	int index;
1815
1816	for (index = end; index >= start; index--)
1817		pci_restore_config_dword(pdev, 4 * index,
1818					 pdev->saved_config_space[index],
1819					 retry, force);
1820}
1821
1822static void pci_restore_config_space(struct pci_dev *pdev)
1823{
1824	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1825		pci_restore_config_space_range(pdev, 10, 15, 0, false);
1826		/* Restore BARs before the command register. */
1827		pci_restore_config_space_range(pdev, 4, 9, 10, false);
1828		pci_restore_config_space_range(pdev, 0, 3, 0, false);
1829	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1830		pci_restore_config_space_range(pdev, 12, 15, 0, false);
1831
1832		/*
1833		 * Force rewriting of prefetch registers to avoid S3 resume
1834		 * issues on Intel PCI bridges that occur when these
1835		 * registers are not explicitly written.
1836		 */
1837		pci_restore_config_space_range(pdev, 9, 11, 0, true);
1838		pci_restore_config_space_range(pdev, 0, 8, 0, false);
1839	} else {
1840		pci_restore_config_space_range(pdev, 0, 15, 0, false);
1841	}
1842}
1843
1844static void pci_restore_rebar_state(struct pci_dev *pdev)
1845{
1846	unsigned int pos, nbars, i;
1847	u32 ctrl;
1848
1849	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1850	if (!pos)
1851		return;
1852
1853	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1854	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
 
1855
1856	for (i = 0; i < nbars; i++, pos += 8) {
1857		struct resource *res;
1858		int bar_idx, size;
1859
1860		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1861		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1862		res = pdev->resource + bar_idx;
1863		size = pci_rebar_bytes_to_size(resource_size(res));
1864		ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1865		ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1866		pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
1867	}
1868}
1869
1870/**
1871 * pci_restore_state - Restore the saved state of a PCI device
1872 * @dev: PCI device that we're dealing with
1873 */
1874void pci_restore_state(struct pci_dev *dev)
1875{
1876	if (!dev->state_saved)
1877		return;
1878
 
 
 
 
 
 
1879	pci_restore_pcie_state(dev);
1880	pci_restore_pasid_state(dev);
1881	pci_restore_pri_state(dev);
1882	pci_restore_ats_state(dev);
1883	pci_restore_vc_state(dev);
1884	pci_restore_rebar_state(dev);
1885	pci_restore_dpc_state(dev);
1886	pci_restore_ptm_state(dev);
1887
1888	pci_aer_clear_status(dev);
1889	pci_restore_aer_state(dev);
1890
1891	pci_restore_config_space(dev);
1892
1893	pci_restore_pcix_state(dev);
1894	pci_restore_msi_state(dev);
1895
1896	/* Restore ACS and IOV configuration state */
1897	pci_enable_acs(dev);
1898	pci_restore_iov_state(dev);
1899
1900	dev->state_saved = false;
1901}
1902EXPORT_SYMBOL(pci_restore_state);
1903
1904struct pci_saved_state {
1905	u32 config_space[16];
1906	struct pci_cap_saved_data cap[];
1907};
1908
1909/**
1910 * pci_store_saved_state - Allocate and return an opaque struct containing
1911 *			   the device saved state.
1912 * @dev: PCI device that we're dealing with
1913 *
1914 * Return NULL if no state or error.
1915 */
1916struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1917{
1918	struct pci_saved_state *state;
1919	struct pci_cap_saved_state *tmp;
1920	struct pci_cap_saved_data *cap;
1921	size_t size;
1922
1923	if (!dev->state_saved)
1924		return NULL;
1925
1926	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1927
1928	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1929		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1930
1931	state = kzalloc(size, GFP_KERNEL);
1932	if (!state)
1933		return NULL;
1934
1935	memcpy(state->config_space, dev->saved_config_space,
1936	       sizeof(state->config_space));
1937
1938	cap = state->cap;
1939	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1940		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1941		memcpy(cap, &tmp->cap, len);
1942		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1943	}
1944	/* Empty cap_save terminates list */
1945
1946	return state;
1947}
1948EXPORT_SYMBOL_GPL(pci_store_saved_state);
1949
1950/**
1951 * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1952 * @dev: PCI device that we're dealing with
1953 * @state: Saved state returned from pci_store_saved_state()
1954 */
1955int pci_load_saved_state(struct pci_dev *dev,
1956			 struct pci_saved_state *state)
1957{
1958	struct pci_cap_saved_data *cap;
1959
1960	dev->state_saved = false;
1961
1962	if (!state)
1963		return 0;
1964
1965	memcpy(dev->saved_config_space, state->config_space,
1966	       sizeof(state->config_space));
1967
1968	cap = state->cap;
1969	while (cap->size) {
1970		struct pci_cap_saved_state *tmp;
1971
1972		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1973		if (!tmp || tmp->cap.size != cap->size)
1974			return -EINVAL;
1975
1976		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1977		cap = (struct pci_cap_saved_data *)((u8 *)cap +
1978		       sizeof(struct pci_cap_saved_data) + cap->size);
1979	}
1980
1981	dev->state_saved = true;
1982	return 0;
1983}
1984EXPORT_SYMBOL_GPL(pci_load_saved_state);
1985
1986/**
1987 * pci_load_and_free_saved_state - Reload the save state pointed to by state,
1988 *				   and free the memory allocated for it.
1989 * @dev: PCI device that we're dealing with
1990 * @state: Pointer to saved state returned from pci_store_saved_state()
1991 */
1992int pci_load_and_free_saved_state(struct pci_dev *dev,
1993				  struct pci_saved_state **state)
1994{
1995	int ret = pci_load_saved_state(dev, *state);
1996	kfree(*state);
1997	*state = NULL;
1998	return ret;
1999}
2000EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
2001
2002int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
2003{
2004	return pci_enable_resources(dev, bars);
2005}
2006
2007static int do_pci_enable_device(struct pci_dev *dev, int bars)
2008{
2009	int err;
2010	struct pci_dev *bridge;
2011	u16 cmd;
2012	u8 pin;
2013
2014	err = pci_set_power_state(dev, PCI_D0);
2015	if (err < 0 && err != -EIO)
2016		return err;
2017
2018	bridge = pci_upstream_bridge(dev);
2019	if (bridge)
2020		pcie_aspm_powersave_config_link(bridge);
2021
2022	err = pcibios_enable_device(dev, bars);
2023	if (err < 0)
2024		return err;
2025	pci_fixup_device(pci_fixup_enable, dev);
2026
2027	if (dev->msi_enabled || dev->msix_enabled)
2028		return 0;
2029
2030	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
2031	if (pin) {
2032		pci_read_config_word(dev, PCI_COMMAND, &cmd);
2033		if (cmd & PCI_COMMAND_INTX_DISABLE)
2034			pci_write_config_word(dev, PCI_COMMAND,
2035					      cmd & ~PCI_COMMAND_INTX_DISABLE);
2036	}
2037
2038	return 0;
2039}
2040
2041/**
2042 * pci_reenable_device - Resume abandoned device
2043 * @dev: PCI device to be resumed
2044 *
2045 * NOTE: This function is a backend of pci_default_resume() and is not supposed
2046 * to be called by normal code, write proper resume handler and use it instead.
2047 */
2048int pci_reenable_device(struct pci_dev *dev)
2049{
2050	if (pci_is_enabled(dev))
2051		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
2052	return 0;
2053}
2054EXPORT_SYMBOL(pci_reenable_device);
2055
2056static void pci_enable_bridge(struct pci_dev *dev)
2057{
2058	struct pci_dev *bridge;
2059	int retval;
2060
2061	bridge = pci_upstream_bridge(dev);
2062	if (bridge)
2063		pci_enable_bridge(bridge);
2064
2065	if (pci_is_enabled(dev)) {
2066		if (!dev->is_busmaster)
2067			pci_set_master(dev);
2068		return;
2069	}
2070
2071	retval = pci_enable_device(dev);
2072	if (retval)
2073		pci_err(dev, "Error enabling bridge (%d), continuing\n",
2074			retval);
2075	pci_set_master(dev);
2076}
2077
2078static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
2079{
2080	struct pci_dev *bridge;
2081	int err;
2082	int i, bars = 0;
2083
2084	/*
2085	 * Power state could be unknown at this point, either due to a fresh
2086	 * boot or a device removal call.  So get the current power state
2087	 * so that things like MSI message writing will behave as expected
2088	 * (e.g. if the device really is in D0 at enable time).
2089	 */
2090	pci_update_current_state(dev, dev->current_state);
2091
2092	if (atomic_inc_return(&dev->enable_cnt) > 1)
2093		return 0;		/* already enabled */
2094
2095	bridge = pci_upstream_bridge(dev);
2096	if (bridge)
2097		pci_enable_bridge(bridge);
2098
2099	/* only skip sriov related */
2100	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2101		if (dev->resource[i].flags & flags)
2102			bars |= (1 << i);
2103	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
2104		if (dev->resource[i].flags & flags)
2105			bars |= (1 << i);
2106
2107	err = do_pci_enable_device(dev, bars);
2108	if (err < 0)
2109		atomic_dec(&dev->enable_cnt);
2110	return err;
2111}
2112
2113/**
2114 * pci_enable_device_io - Initialize a device for use with IO space
2115 * @dev: PCI device to be initialized
2116 *
2117 * Initialize device before it's used by a driver. Ask low-level code
2118 * to enable I/O resources. Wake up the device if it was suspended.
2119 * Beware, this function can fail.
2120 */
2121int pci_enable_device_io(struct pci_dev *dev)
2122{
2123	return pci_enable_device_flags(dev, IORESOURCE_IO);
2124}
2125EXPORT_SYMBOL(pci_enable_device_io);
2126
2127/**
2128 * pci_enable_device_mem - Initialize a device for use with Memory space
2129 * @dev: PCI device to be initialized
2130 *
2131 * Initialize device before it's used by a driver. Ask low-level code
2132 * to enable Memory resources. Wake up the device if it was suspended.
2133 * Beware, this function can fail.
2134 */
2135int pci_enable_device_mem(struct pci_dev *dev)
2136{
2137	return pci_enable_device_flags(dev, IORESOURCE_MEM);
2138}
2139EXPORT_SYMBOL(pci_enable_device_mem);
2140
2141/**
2142 * pci_enable_device - Initialize device before it's used by a driver.
2143 * @dev: PCI device to be initialized
2144 *
2145 * Initialize device before it's used by a driver. Ask low-level code
2146 * to enable I/O and memory. Wake up the device if it was suspended.
2147 * Beware, this function can fail.
2148 *
2149 * Note we don't actually enable the device many times if we call
2150 * this function repeatedly (we just increment the count).
2151 */
2152int pci_enable_device(struct pci_dev *dev)
2153{
2154	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
2155}
2156EXPORT_SYMBOL(pci_enable_device);
2157
2158/*
2159 * pcibios_device_add - provide arch specific hooks when adding device dev
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2160 * @dev: the PCI device being added
2161 *
2162 * Permits the platform to provide architecture specific functionality when
2163 * devices are added. This is the default implementation. Architecture
2164 * implementations can override this.
2165 */
2166int __weak pcibios_device_add(struct pci_dev *dev)
2167{
2168	return 0;
2169}
2170
2171/**
2172 * pcibios_release_device - provide arch specific hooks when releasing
2173 *			    device dev
2174 * @dev: the PCI device being released
2175 *
2176 * Permits the platform to provide architecture specific functionality when
2177 * devices are released. This is the default implementation. Architecture
2178 * implementations can override this.
2179 */
2180void __weak pcibios_release_device(struct pci_dev *dev) {}
2181
2182/**
2183 * pcibios_disable_device - disable arch specific PCI resources for device dev
2184 * @dev: the PCI device to disable
2185 *
2186 * Disables architecture specific PCI resources for the device. This
2187 * is the default implementation. Architecture implementations can
2188 * override this.
2189 */
2190void __weak pcibios_disable_device(struct pci_dev *dev) {}
2191
 
 
 
 
 
 
 
 
 
 
 
2192static void do_pci_disable_device(struct pci_dev *dev)
2193{
2194	u16 pci_command;
2195
2196	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
2197	if (pci_command & PCI_COMMAND_MASTER) {
2198		pci_command &= ~PCI_COMMAND_MASTER;
2199		pci_write_config_word(dev, PCI_COMMAND, pci_command);
2200	}
2201
2202	pcibios_disable_device(dev);
2203}
2204
2205/**
2206 * pci_disable_enabled_device - Disable device without updating enable_cnt
2207 * @dev: PCI device to disable
2208 *
2209 * NOTE: This function is a backend of PCI power management routines and is
2210 * not supposed to be called drivers.
2211 */
2212void pci_disable_enabled_device(struct pci_dev *dev)
2213{
2214	if (pci_is_enabled(dev))
2215		do_pci_disable_device(dev);
2216}
2217
2218/**
2219 * pci_disable_device - Disable PCI device after use
2220 * @dev: PCI device to be disabled
2221 *
2222 * Signal to the system that the PCI device is not in use by the system
2223 * anymore.  This only involves disabling PCI bus-mastering, if active.
2224 *
2225 * Note we don't actually disable the device until all callers of
2226 * pci_enable_device() have called pci_disable_device().
2227 */
2228void pci_disable_device(struct pci_dev *dev)
2229{
2230	struct pci_devres *dr;
2231
2232	dr = find_pci_dr(dev);
2233	if (dr)
2234		dr->enabled = 0;
2235
2236	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
2237		      "disabling already-disabled device");
2238
2239	if (atomic_dec_return(&dev->enable_cnt) != 0)
2240		return;
2241
2242	do_pci_disable_device(dev);
2243
2244	dev->is_busmaster = 0;
2245}
2246EXPORT_SYMBOL(pci_disable_device);
2247
2248/**
2249 * pcibios_set_pcie_reset_state - set reset state for device dev
2250 * @dev: the PCIe device reset
2251 * @state: Reset state to enter into
2252 *
2253 * Set the PCIe reset state for the device. This is the default
2254 * implementation. Architecture implementations can override this.
2255 */
2256int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
2257					enum pcie_reset_state state)
2258{
2259	return -EINVAL;
2260}
2261
2262/**
2263 * pci_set_pcie_reset_state - set reset state for device dev
2264 * @dev: the PCIe device reset
2265 * @state: Reset state to enter into
2266 *
2267 * Sets the PCI reset state for the device.
2268 */
2269int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
2270{
2271	return pcibios_set_pcie_reset_state(dev, state);
2272}
2273EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
2274
2275#ifdef CONFIG_PCIEAER
2276void pcie_clear_device_status(struct pci_dev *dev)
2277{
2278	u16 sta;
2279
2280	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
2281	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
2282}
2283#endif
2284
2285/**
2286 * pcie_clear_root_pme_status - Clear root port PME interrupt status.
2287 * @dev: PCIe root port or event collector.
2288 */
2289void pcie_clear_root_pme_status(struct pci_dev *dev)
2290{
2291	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
2292}
2293
2294/**
2295 * pci_check_pme_status - Check if given device has generated PME.
2296 * @dev: Device to check.
2297 *
2298 * Check the PME status of the device and if set, clear it and clear PME enable
2299 * (if set).  Return 'true' if PME status and PME enable were both set or
2300 * 'false' otherwise.
2301 */
2302bool pci_check_pme_status(struct pci_dev *dev)
2303{
2304	int pmcsr_pos;
2305	u16 pmcsr;
2306	bool ret = false;
2307
2308	if (!dev->pm_cap)
2309		return false;
2310
2311	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
2312	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
2313	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
2314		return false;
2315
2316	/* Clear PME status. */
2317	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2318	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
2319		/* Disable PME to avoid interrupt flood. */
2320		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2321		ret = true;
2322	}
2323
2324	pci_write_config_word(dev, pmcsr_pos, pmcsr);
2325
2326	return ret;
2327}
2328
2329/**
2330 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
2331 * @dev: Device to handle.
2332 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
2333 *
2334 * Check if @dev has generated PME and queue a resume request for it in that
2335 * case.
2336 */
2337static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
2338{
2339	if (pme_poll_reset && dev->pme_poll)
2340		dev->pme_poll = false;
2341
2342	if (pci_check_pme_status(dev)) {
2343		pci_wakeup_event(dev);
2344		pm_request_resume(&dev->dev);
2345	}
2346	return 0;
2347}
2348
2349/**
2350 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
2351 * @bus: Top bus of the subtree to walk.
2352 */
2353void pci_pme_wakeup_bus(struct pci_bus *bus)
2354{
2355	if (bus)
2356		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
2357}
2358
2359
2360/**
2361 * pci_pme_capable - check the capability of PCI device to generate PME#
2362 * @dev: PCI device to handle.
2363 * @state: PCI state from which device will issue PME#.
2364 */
2365bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
2366{
2367	if (!dev->pm_cap)
2368		return false;
2369
2370	return !!(dev->pme_support & (1 << state));
2371}
2372EXPORT_SYMBOL(pci_pme_capable);
2373
2374static void pci_pme_list_scan(struct work_struct *work)
2375{
2376	struct pci_pme_device *pme_dev, *n;
2377
2378	mutex_lock(&pci_pme_list_mutex);
2379	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
2380		struct pci_dev *pdev = pme_dev->dev;
2381
2382		if (pdev->pme_poll) {
2383			struct pci_dev *bridge = pdev->bus->self;
2384			struct device *dev = &pdev->dev;
2385			struct device *bdev = bridge ? &bridge->dev : NULL;
2386			int bref = 0;
2387
 
2388			/*
2389			 * If we have a bridge, it should be in an active/D0
2390			 * state or the configuration space of subordinate
2391			 * devices may not be accessible or stable over the
2392			 * course of the call.
2393			 */
2394			if (bdev) {
2395				bref = pm_runtime_get_if_active(bdev);
2396				if (!bref)
2397					continue;
2398
2399				if (bridge->current_state != PCI_D0)
2400					goto put_bridge;
2401			}
2402
2403			/*
2404			 * The device itself should be suspended but config
2405			 * space must be accessible, therefore it cannot be in
2406			 * D3cold.
2407			 */
2408			if (pm_runtime_suspended(dev) &&
2409			    pdev->current_state != PCI_D3cold)
2410				pci_pme_wakeup(pdev, NULL);
2411
2412put_bridge:
2413			if (bref > 0)
2414				pm_runtime_put(bdev);
2415		} else {
2416			list_del(&pme_dev->list);
2417			kfree(pme_dev);
2418		}
2419	}
2420	if (!list_empty(&pci_pme_list))
2421		queue_delayed_work(system_freezable_wq, &pci_pme_work,
2422				   msecs_to_jiffies(PME_TIMEOUT));
2423	mutex_unlock(&pci_pme_list_mutex);
2424}
2425
2426static void __pci_pme_active(struct pci_dev *dev, bool enable)
2427{
2428	u16 pmcsr;
2429
2430	if (!dev->pme_support)
2431		return;
2432
2433	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2434	/* Clear PME_Status by writing 1 to it and enable PME# */
2435	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2436	if (!enable)
2437		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2438
2439	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2440}
2441
2442/**
2443 * pci_pme_restore - Restore PME configuration after config space restore.
2444 * @dev: PCI device to update.
2445 */
2446void pci_pme_restore(struct pci_dev *dev)
2447{
2448	u16 pmcsr;
2449
2450	if (!dev->pme_support)
2451		return;
2452
2453	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2454	if (dev->wakeup_prepared) {
2455		pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2456		pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2457	} else {
2458		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2459		pmcsr |= PCI_PM_CTRL_PME_STATUS;
2460	}
2461	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2462}
2463
2464/**
2465 * pci_pme_active - enable or disable PCI device's PME# function
2466 * @dev: PCI device to handle.
2467 * @enable: 'true' to enable PME# generation; 'false' to disable it.
2468 *
2469 * The caller must verify that the device is capable of generating PME# before
2470 * calling this function with @enable equal to 'true'.
2471 */
2472void pci_pme_active(struct pci_dev *dev, bool enable)
2473{
2474	__pci_pme_active(dev, enable);
2475
2476	/*
2477	 * PCI (as opposed to PCIe) PME requires that the device have
2478	 * its PME# line hooked up correctly. Not all hardware vendors
2479	 * do this, so the PME never gets delivered and the device
2480	 * remains asleep. The easiest way around this is to
2481	 * periodically walk the list of suspended devices and check
2482	 * whether any have their PME flag set. The assumption is that
2483	 * we'll wake up often enough anyway that this won't be a huge
2484	 * hit, and the power savings from the devices will still be a
2485	 * win.
2486	 *
2487	 * Although PCIe uses in-band PME message instead of PME# line
2488	 * to report PME, PME does not work for some PCIe devices in
2489	 * reality.  For example, there are devices that set their PME
2490	 * status bits, but don't really bother to send a PME message;
2491	 * there are PCI Express Root Ports that don't bother to
2492	 * trigger interrupts when they receive PME messages from the
2493	 * devices below.  So PME poll is used for PCIe devices too.
2494	 */
2495
2496	if (dev->pme_poll) {
2497		struct pci_pme_device *pme_dev;
2498		if (enable) {
2499			pme_dev = kmalloc(sizeof(struct pci_pme_device),
2500					  GFP_KERNEL);
2501			if (!pme_dev) {
2502				pci_warn(dev, "can't enable PME#\n");
2503				return;
2504			}
2505			pme_dev->dev = dev;
2506			mutex_lock(&pci_pme_list_mutex);
2507			list_add(&pme_dev->list, &pci_pme_list);
2508			if (list_is_singular(&pci_pme_list))
2509				queue_delayed_work(system_freezable_wq,
2510						   &pci_pme_work,
2511						   msecs_to_jiffies(PME_TIMEOUT));
2512			mutex_unlock(&pci_pme_list_mutex);
2513		} else {
2514			mutex_lock(&pci_pme_list_mutex);
2515			list_for_each_entry(pme_dev, &pci_pme_list, list) {
2516				if (pme_dev->dev == dev) {
2517					list_del(&pme_dev->list);
2518					kfree(pme_dev);
2519					break;
2520				}
2521			}
2522			mutex_unlock(&pci_pme_list_mutex);
2523		}
2524	}
2525
2526	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2527}
2528EXPORT_SYMBOL(pci_pme_active);
2529
2530/**
2531 * __pci_enable_wake - enable PCI device as wakeup event source
2532 * @dev: PCI device affected
2533 * @state: PCI state from which device will issue wakeup events
2534 * @enable: True to enable event generation; false to disable
2535 *
2536 * This enables the device as a wakeup event source, or disables it.
2537 * When such events involves platform-specific hooks, those hooks are
2538 * called automatically by this routine.
2539 *
2540 * Devices with legacy power management (no standard PCI PM capabilities)
2541 * always require such platform hooks.
2542 *
2543 * RETURN VALUE:
2544 * 0 is returned on success
2545 * -EINVAL is returned if device is not supposed to wake up the system
2546 * Error code depending on the platform is returned if both the platform and
2547 * the native mechanism fail to enable the generation of wake-up events
2548 */
2549static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2550{
2551	int ret = 0;
2552
2553	/*
2554	 * Bridges that are not power-manageable directly only signal
2555	 * wakeup on behalf of subordinate devices which is set up
2556	 * elsewhere, so skip them. However, bridges that are
2557	 * power-manageable may signal wakeup for themselves (for example,
2558	 * on a hotplug event) and they need to be covered here.
2559	 */
2560	if (!pci_power_manageable(dev))
2561		return 0;
2562
2563	/* Don't do the same thing twice in a row for one device. */
2564	if (!!enable == !!dev->wakeup_prepared)
2565		return 0;
2566
2567	/*
2568	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2569	 * Anderson we should be doing PME# wake enable followed by ACPI wake
2570	 * enable.  To disable wake-up we call the platform first, for symmetry.
2571	 */
2572
2573	if (enable) {
2574		int error;
2575
2576		/*
2577		 * Enable PME signaling if the device can signal PME from
2578		 * D3cold regardless of whether or not it can signal PME from
2579		 * the current target state, because that will allow it to
2580		 * signal PME when the hierarchy above it goes into D3cold and
2581		 * the device itself ends up in D3cold as a result of that.
2582		 */
2583		if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2584			pci_pme_active(dev, true);
2585		else
2586			ret = 1;
2587		error = platform_pci_set_wakeup(dev, true);
2588		if (ret)
2589			ret = error;
2590		if (!ret)
2591			dev->wakeup_prepared = true;
2592	} else {
2593		platform_pci_set_wakeup(dev, false);
2594		pci_pme_active(dev, false);
2595		dev->wakeup_prepared = false;
2596	}
2597
2598	return ret;
2599}
2600
2601/**
2602 * pci_enable_wake - change wakeup settings for a PCI device
2603 * @pci_dev: Target device
2604 * @state: PCI state from which device will issue wakeup events
2605 * @enable: Whether or not to enable event generation
2606 *
2607 * If @enable is set, check device_may_wakeup() for the device before calling
2608 * __pci_enable_wake() for it.
2609 */
2610int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2611{
2612	if (enable && !device_may_wakeup(&pci_dev->dev))
2613		return -EINVAL;
2614
2615	return __pci_enable_wake(pci_dev, state, enable);
2616}
2617EXPORT_SYMBOL(pci_enable_wake);
2618
2619/**
2620 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2621 * @dev: PCI device to prepare
2622 * @enable: True to enable wake-up event generation; false to disable
2623 *
2624 * Many drivers want the device to wake up the system from D3_hot or D3_cold
2625 * and this function allows them to set that up cleanly - pci_enable_wake()
2626 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2627 * ordering constraints.
2628 *
2629 * This function only returns error code if the device is not allowed to wake
2630 * up the system from sleep or it is not capable of generating PME# from both
2631 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2632 */
2633int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2634{
2635	return pci_pme_capable(dev, PCI_D3cold) ?
2636			pci_enable_wake(dev, PCI_D3cold, enable) :
2637			pci_enable_wake(dev, PCI_D3hot, enable);
2638}
2639EXPORT_SYMBOL(pci_wake_from_d3);
2640
2641/**
2642 * pci_target_state - find an appropriate low power state for a given PCI dev
2643 * @dev: PCI device
2644 * @wakeup: Whether or not wakeup functionality will be enabled for the device.
2645 *
2646 * Use underlying platform code to find a supported low power state for @dev.
2647 * If the platform can't manage @dev, return the deepest state from which it
2648 * can generate wake events, based on any available PME info.
2649 */
2650static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2651{
 
 
2652	if (platform_pci_power_manageable(dev)) {
2653		/*
2654		 * Call the platform to find the target state for the device.
2655		 */
2656		pci_power_t state = platform_pci_choose_state(dev);
2657
2658		switch (state) {
2659		case PCI_POWER_ERROR:
2660		case PCI_UNKNOWN:
2661			return PCI_D3hot;
2662
2663		case PCI_D1:
2664		case PCI_D2:
2665			if (pci_no_d1d2(dev))
2666				return PCI_D3hot;
 
 
 
2667		}
2668
2669		return state;
2670	}
2671
 
 
 
2672	/*
2673	 * If the device is in D3cold even though it's not power-manageable by
2674	 * the platform, it may have been powered down by non-standard means.
2675	 * Best to let it slumber.
2676	 */
2677	if (dev->current_state == PCI_D3cold)
2678		return PCI_D3cold;
2679	else if (!dev->pm_cap)
2680		return PCI_D0;
2681
2682	if (wakeup && dev->pme_support) {
2683		pci_power_t state = PCI_D3hot;
2684
2685		/*
2686		 * Find the deepest state from which the device can generate
2687		 * PME#.
2688		 */
2689		while (state && !(dev->pme_support & (1 << state)))
2690			state--;
2691
2692		if (state)
2693			return state;
2694		else if (dev->pme_support & 1)
2695			return PCI_D0;
2696	}
2697
2698	return PCI_D3hot;
2699}
2700
2701/**
2702 * pci_prepare_to_sleep - prepare PCI device for system-wide transition
2703 *			  into a sleep state
2704 * @dev: Device to handle.
2705 *
2706 * Choose the power state appropriate for the device depending on whether
2707 * it can wake up the system and/or is power manageable by the platform
2708 * (PCI_D3hot is the default) and put the device into that state.
2709 */
2710int pci_prepare_to_sleep(struct pci_dev *dev)
2711{
2712	bool wakeup = device_may_wakeup(&dev->dev);
2713	pci_power_t target_state = pci_target_state(dev, wakeup);
2714	int error;
2715
2716	if (target_state == PCI_POWER_ERROR)
2717		return -EIO;
2718
 
 
 
 
 
 
 
 
 
 
2719	pci_enable_wake(dev, target_state, wakeup);
2720
2721	error = pci_set_power_state(dev, target_state);
2722
2723	if (error)
2724		pci_enable_wake(dev, target_state, false);
 
 
2725
2726	return error;
2727}
2728EXPORT_SYMBOL(pci_prepare_to_sleep);
2729
2730/**
2731 * pci_back_from_sleep - turn PCI device on during system-wide transition
2732 *			 into working state
2733 * @dev: Device to handle.
2734 *
2735 * Disable device's system wake-up capability and put it into D0.
2736 */
2737int pci_back_from_sleep(struct pci_dev *dev)
2738{
2739	int ret = pci_set_power_state(dev, PCI_D0);
2740
2741	if (ret)
2742		return ret;
2743
2744	pci_enable_wake(dev, PCI_D0, false);
2745	return 0;
2746}
2747EXPORT_SYMBOL(pci_back_from_sleep);
2748
2749/**
2750 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2751 * @dev: PCI device being suspended.
2752 *
2753 * Prepare @dev to generate wake-up events at run time and put it into a low
2754 * power state.
2755 */
2756int pci_finish_runtime_suspend(struct pci_dev *dev)
2757{
2758	pci_power_t target_state;
2759	int error;
2760
2761	target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
2762	if (target_state == PCI_POWER_ERROR)
2763		return -EIO;
2764
 
 
 
 
 
 
 
 
 
 
 
 
2765	__pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
2766
2767	error = pci_set_power_state(dev, target_state);
2768
2769	if (error)
2770		pci_enable_wake(dev, target_state, false);
 
 
 
2771
2772	return error;
2773}
2774
2775/**
2776 * pci_dev_run_wake - Check if device can generate run-time wake-up events.
2777 * @dev: Device to check.
2778 *
2779 * Return true if the device itself is capable of generating wake-up events
2780 * (through the platform or using the native PCIe PME) or if the device supports
2781 * PME and one of its upstream bridges can generate wake-up events.
2782 */
2783bool pci_dev_run_wake(struct pci_dev *dev)
2784{
2785	struct pci_bus *bus = dev->bus;
2786
2787	if (!dev->pme_support)
2788		return false;
2789
2790	/* PME-capable in principle, but not from the target power state */
2791	if (!pci_pme_capable(dev, pci_target_state(dev, true)))
2792		return false;
2793
2794	if (device_can_wakeup(&dev->dev))
2795		return true;
2796
2797	while (bus->parent) {
2798		struct pci_dev *bridge = bus->self;
2799
2800		if (device_can_wakeup(&bridge->dev))
2801			return true;
2802
2803		bus = bus->parent;
2804	}
2805
2806	/* We have reached the root bus. */
2807	if (bus->bridge)
2808		return device_can_wakeup(bus->bridge);
2809
2810	return false;
2811}
2812EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2813
2814/**
2815 * pci_dev_need_resume - Check if it is necessary to resume the device.
2816 * @pci_dev: Device to check.
2817 *
2818 * Return 'true' if the device is not runtime-suspended or it has to be
2819 * reconfigured due to wakeup settings difference between system and runtime
2820 * suspend, or the current power state of it is not suitable for the upcoming
2821 * (system-wide) transition.
2822 */
2823bool pci_dev_need_resume(struct pci_dev *pci_dev)
2824{
2825	struct device *dev = &pci_dev->dev;
2826	pci_power_t target_state;
2827
2828	if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev))
2829		return true;
2830
2831	target_state = pci_target_state(pci_dev, device_may_wakeup(dev));
2832
2833	/*
2834	 * If the earlier platform check has not triggered, D3cold is just power
2835	 * removal on top of D3hot, so no need to resume the device in that
2836	 * case.
2837	 */
2838	return target_state != pci_dev->current_state &&
2839		target_state != PCI_D3cold &&
2840		pci_dev->current_state != PCI_D3hot;
2841}
2842
2843/**
2844 * pci_dev_adjust_pme - Adjust PME setting for a suspended device.
2845 * @pci_dev: Device to check.
2846 *
2847 * If the device is suspended and it is not configured for system wakeup,
2848 * disable PME for it to prevent it from waking up the system unnecessarily.
2849 *
2850 * Note that if the device's power state is D3cold and the platform check in
2851 * pci_dev_need_resume() has not triggered, the device's configuration need not
2852 * be changed.
2853 */
2854void pci_dev_adjust_pme(struct pci_dev *pci_dev)
2855{
2856	struct device *dev = &pci_dev->dev;
2857
2858	spin_lock_irq(&dev->power.lock);
2859
2860	if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
2861	    pci_dev->current_state < PCI_D3cold)
2862		__pci_pme_active(pci_dev, false);
2863
2864	spin_unlock_irq(&dev->power.lock);
2865}
2866
2867/**
2868 * pci_dev_complete_resume - Finalize resume from system sleep for a device.
2869 * @pci_dev: Device to handle.
2870 *
2871 * If the device is runtime suspended and wakeup-capable, enable PME for it as
2872 * it might have been disabled during the prepare phase of system suspend if
2873 * the device was not configured for system wakeup.
2874 */
2875void pci_dev_complete_resume(struct pci_dev *pci_dev)
2876{
2877	struct device *dev = &pci_dev->dev;
2878
2879	if (!pci_dev_run_wake(pci_dev))
2880		return;
2881
2882	spin_lock_irq(&dev->power.lock);
2883
2884	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2885		__pci_pme_active(pci_dev, true);
2886
2887	spin_unlock_irq(&dev->power.lock);
2888}
2889
2890/**
2891 * pci_choose_state - Choose the power state of a PCI device.
2892 * @dev: Target PCI device.
2893 * @state: Target state for the whole system.
2894 *
2895 * Returns PCI power state suitable for @dev and @state.
2896 */
2897pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
2898{
2899	if (state.event == PM_EVENT_ON)
2900		return PCI_D0;
2901
2902	return pci_target_state(dev, false);
2903}
2904EXPORT_SYMBOL(pci_choose_state);
2905
2906void pci_config_pm_runtime_get(struct pci_dev *pdev)
2907{
2908	struct device *dev = &pdev->dev;
2909	struct device *parent = dev->parent;
2910
2911	if (parent)
2912		pm_runtime_get_sync(parent);
2913	pm_runtime_get_noresume(dev);
2914	/*
2915	 * pdev->current_state is set to PCI_D3cold during suspending,
2916	 * so wait until suspending completes
2917	 */
2918	pm_runtime_barrier(dev);
2919	/*
2920	 * Only need to resume devices in D3cold, because config
2921	 * registers are still accessible for devices suspended but
2922	 * not in D3cold.
2923	 */
2924	if (pdev->current_state == PCI_D3cold)
2925		pm_runtime_resume(dev);
2926}
2927
2928void pci_config_pm_runtime_put(struct pci_dev *pdev)
2929{
2930	struct device *dev = &pdev->dev;
2931	struct device *parent = dev->parent;
2932
2933	pm_runtime_put(dev);
2934	if (parent)
2935		pm_runtime_put_sync(parent);
2936}
2937
2938static const struct dmi_system_id bridge_d3_blacklist[] = {
2939#ifdef CONFIG_X86
2940	{
2941		/*
2942		 * Gigabyte X299 root port is not marked as hotplug capable
2943		 * which allows Linux to power manage it.  However, this
2944		 * confuses the BIOS SMI handler so don't power manage root
2945		 * ports on that system.
2946		 */
2947		.ident = "X299 DESIGNARE EX-CF",
2948		.matches = {
2949			DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2950			DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2951		},
2952	},
2953	{
2954		/*
2955		 * Downstream device is not accessible after putting a root port
2956		 * into D3cold and back into D0 on Elo Continental Z2 board
2957		 */
2958		.ident = "Elo Continental Z2",
2959		.matches = {
2960			DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
2961			DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
2962			DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
2963		},
2964	},
2965#endif
2966	{ }
2967};
2968
2969/**
2970 * pci_bridge_d3_possible - Is it possible to put the bridge into D3
2971 * @bridge: Bridge to check
2972 *
2973 * This function checks if it is possible to move the bridge to D3.
2974 * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
2975 */
2976bool pci_bridge_d3_possible(struct pci_dev *bridge)
2977{
2978	if (!pci_is_pcie(bridge))
2979		return false;
2980
2981	switch (pci_pcie_type(bridge)) {
2982	case PCI_EXP_TYPE_ROOT_PORT:
2983	case PCI_EXP_TYPE_UPSTREAM:
2984	case PCI_EXP_TYPE_DOWNSTREAM:
2985		if (pci_bridge_d3_disable)
2986			return false;
2987
2988		/*
2989		 * Hotplug ports handled by firmware in System Management Mode
2990		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
2991		 */
2992		if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
2993			return false;
2994
2995		if (pci_bridge_d3_force)
2996			return true;
2997
2998		/* Even the oldest 2010 Thunderbolt controller supports D3. */
2999		if (bridge->is_thunderbolt)
3000			return true;
3001
3002		/* Platform might know better if the bridge supports D3 */
3003		if (platform_pci_bridge_d3(bridge))
3004			return true;
3005
3006		/*
3007		 * Hotplug ports handled natively by the OS were not validated
3008		 * by vendors for runtime D3 at least until 2018 because there
3009		 * was no OS support.
3010		 */
3011		if (bridge->is_hotplug_bridge)
3012			return false;
3013
3014		if (dmi_check_system(bridge_d3_blacklist))
3015			return false;
3016
3017		/*
3018		 * It should be safe to put PCIe ports from 2015 or newer
3019		 * to D3.
3020		 */
3021		if (dmi_get_bios_year() >= 2015)
3022			return true;
3023		break;
3024	}
3025
3026	return false;
3027}
3028
3029static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
3030{
3031	bool *d3cold_ok = data;
3032
3033	if (/* The device needs to be allowed to go D3cold ... */
3034	    dev->no_d3cold || !dev->d3cold_allowed ||
3035
3036	    /* ... and if it is wakeup capable to do so from D3cold. */
3037	    (device_may_wakeup(&dev->dev) &&
3038	     !pci_pme_capable(dev, PCI_D3cold)) ||
3039
3040	    /* If it is a bridge it must be allowed to go to D3. */
3041	    !pci_power_manageable(dev))
3042
3043		*d3cold_ok = false;
3044
3045	return !*d3cold_ok;
3046}
3047
3048/*
3049 * pci_bridge_d3_update - Update bridge D3 capabilities
3050 * @dev: PCI device which is changed
3051 *
3052 * Update upstream bridge PM capabilities accordingly depending on if the
3053 * device PM configuration was changed or the device is being removed.  The
3054 * change is also propagated upstream.
3055 */
3056void pci_bridge_d3_update(struct pci_dev *dev)
3057{
3058	bool remove = !device_is_registered(&dev->dev);
3059	struct pci_dev *bridge;
3060	bool d3cold_ok = true;
3061
3062	bridge = pci_upstream_bridge(dev);
3063	if (!bridge || !pci_bridge_d3_possible(bridge))
3064		return;
3065
3066	/*
3067	 * If D3 is currently allowed for the bridge, removing one of its
3068	 * children won't change that.
3069	 */
3070	if (remove && bridge->bridge_d3)
3071		return;
3072
3073	/*
3074	 * If D3 is currently allowed for the bridge and a child is added or
3075	 * changed, disallowance of D3 can only be caused by that child, so
3076	 * we only need to check that single device, not any of its siblings.
3077	 *
3078	 * If D3 is currently not allowed for the bridge, checking the device
3079	 * first may allow us to skip checking its siblings.
3080	 */
3081	if (!remove)
3082		pci_dev_check_d3cold(dev, &d3cold_ok);
3083
3084	/*
3085	 * If D3 is currently not allowed for the bridge, this may be caused
3086	 * either by the device being changed/removed or any of its siblings,
3087	 * so we need to go through all children to find out if one of them
3088	 * continues to block D3.
3089	 */
3090	if (d3cold_ok && !bridge->bridge_d3)
3091		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
3092			     &d3cold_ok);
3093
3094	if (bridge->bridge_d3 != d3cold_ok) {
3095		bridge->bridge_d3 = d3cold_ok;
3096		/* Propagate change to upstream bridges */
3097		pci_bridge_d3_update(bridge);
3098	}
3099}
3100
3101/**
3102 * pci_d3cold_enable - Enable D3cold for device
3103 * @dev: PCI device to handle
3104 *
3105 * This function can be used in drivers to enable D3cold from the device
3106 * they handle.  It also updates upstream PCI bridge PM capabilities
3107 * accordingly.
3108 */
3109void pci_d3cold_enable(struct pci_dev *dev)
3110{
3111	if (dev->no_d3cold) {
3112		dev->no_d3cold = false;
3113		pci_bridge_d3_update(dev);
3114	}
3115}
3116EXPORT_SYMBOL_GPL(pci_d3cold_enable);
3117
3118/**
3119 * pci_d3cold_disable - Disable D3cold for device
3120 * @dev: PCI device to handle
3121 *
3122 * This function can be used in drivers to disable D3cold from the device
3123 * they handle.  It also updates upstream PCI bridge PM capabilities
3124 * accordingly.
3125 */
3126void pci_d3cold_disable(struct pci_dev *dev)
3127{
3128	if (!dev->no_d3cold) {
3129		dev->no_d3cold = true;
3130		pci_bridge_d3_update(dev);
3131	}
3132}
3133EXPORT_SYMBOL_GPL(pci_d3cold_disable);
3134
3135/**
3136 * pci_pm_init - Initialize PM functions of given PCI device
3137 * @dev: PCI device to handle.
3138 */
3139void pci_pm_init(struct pci_dev *dev)
3140{
3141	int pm;
3142	u16 status;
3143	u16 pmc;
3144
3145	pm_runtime_forbid(&dev->dev);
3146	pm_runtime_set_active(&dev->dev);
3147	pm_runtime_enable(&dev->dev);
3148	device_enable_async_suspend(&dev->dev);
3149	dev->wakeup_prepared = false;
3150
3151	dev->pm_cap = 0;
3152	dev->pme_support = 0;
3153
3154	/* find PCI PM capability in list */
3155	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
3156	if (!pm)
3157		return;
3158	/* Check device's ability to generate PME# */
3159	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
3160
3161	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
3162		pci_err(dev, "unsupported PM cap regs version (%u)\n",
3163			pmc & PCI_PM_CAP_VER_MASK);
3164		return;
3165	}
3166
3167	dev->pm_cap = pm;
3168	dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
3169	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
3170	dev->bridge_d3 = pci_bridge_d3_possible(dev);
3171	dev->d3cold_allowed = true;
3172
3173	dev->d1_support = false;
3174	dev->d2_support = false;
3175	if (!pci_no_d1d2(dev)) {
3176		if (pmc & PCI_PM_CAP_D1)
3177			dev->d1_support = true;
3178		if (pmc & PCI_PM_CAP_D2)
3179			dev->d2_support = true;
3180
3181		if (dev->d1_support || dev->d2_support)
3182			pci_info(dev, "supports%s%s\n",
3183				   dev->d1_support ? " D1" : "",
3184				   dev->d2_support ? " D2" : "");
3185	}
3186
3187	pmc &= PCI_PM_CAP_PME_MASK;
3188	if (pmc) {
3189		pci_info(dev, "PME# supported from%s%s%s%s%s\n",
3190			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
3191			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
3192			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
3193			 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
3194			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
3195		dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc);
3196		dev->pme_poll = true;
3197		/*
3198		 * Make device's PM flags reflect the wake-up capability, but
3199		 * let the user space enable it to wake up the system as needed.
3200		 */
3201		device_set_wakeup_capable(&dev->dev, true);
3202		/* Disable the PME# generation functionality */
3203		pci_pme_active(dev, false);
3204	}
3205
3206	pci_read_config_word(dev, PCI_STATUS, &status);
3207	if (status & PCI_STATUS_IMM_READY)
3208		dev->imm_ready = 1;
3209}
3210
3211static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
3212{
3213	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
3214
3215	switch (prop) {
3216	case PCI_EA_P_MEM:
3217	case PCI_EA_P_VF_MEM:
3218		flags |= IORESOURCE_MEM;
3219		break;
3220	case PCI_EA_P_MEM_PREFETCH:
3221	case PCI_EA_P_VF_MEM_PREFETCH:
3222		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
3223		break;
3224	case PCI_EA_P_IO:
3225		flags |= IORESOURCE_IO;
3226		break;
3227	default:
3228		return 0;
3229	}
3230
3231	return flags;
3232}
3233
3234static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
3235					    u8 prop)
3236{
3237	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
3238		return &dev->resource[bei];
3239#ifdef CONFIG_PCI_IOV
3240	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
3241		 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
3242		return &dev->resource[PCI_IOV_RESOURCES +
3243				      bei - PCI_EA_BEI_VF_BAR0];
3244#endif
3245	else if (bei == PCI_EA_BEI_ROM)
3246		return &dev->resource[PCI_ROM_RESOURCE];
3247	else
3248		return NULL;
3249}
3250
3251/* Read an Enhanced Allocation (EA) entry */
3252static int pci_ea_read(struct pci_dev *dev, int offset)
3253{
3254	struct resource *res;
3255	const char *res_name;
3256	int ent_size, ent_offset = offset;
3257	resource_size_t start, end;
3258	unsigned long flags;
3259	u32 dw0, bei, base, max_offset;
3260	u8 prop;
3261	bool support_64 = (sizeof(resource_size_t) >= 8);
3262
3263	pci_read_config_dword(dev, ent_offset, &dw0);
3264	ent_offset += 4;
3265
3266	/* Entry size field indicates DWORDs after 1st */
3267	ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2;
3268
3269	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
3270		goto out;
3271
3272	bei = FIELD_GET(PCI_EA_BEI, dw0);
3273	prop = FIELD_GET(PCI_EA_PP, dw0);
3274
3275	/*
3276	 * If the Property is in the reserved range, try the Secondary
3277	 * Property instead.
3278	 */
3279	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
3280		prop = FIELD_GET(PCI_EA_SP, dw0);
3281	if (prop > PCI_EA_P_BRIDGE_IO)
3282		goto out;
3283
3284	res = pci_ea_get_resource(dev, bei, prop);
3285	res_name = pci_resource_name(dev, bei);
3286	if (!res) {
3287		pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
3288		goto out;
3289	}
3290
3291	flags = pci_ea_flags(dev, prop);
3292	if (!flags) {
3293		pci_err(dev, "Unsupported EA properties: %#x\n", prop);
3294		goto out;
3295	}
3296
3297	/* Read Base */
3298	pci_read_config_dword(dev, ent_offset, &base);
3299	start = (base & PCI_EA_FIELD_MASK);
3300	ent_offset += 4;
3301
3302	/* Read MaxOffset */
3303	pci_read_config_dword(dev, ent_offset, &max_offset);
3304	ent_offset += 4;
3305
3306	/* Read Base MSBs (if 64-bit entry) */
3307	if (base & PCI_EA_IS_64) {
3308		u32 base_upper;
3309
3310		pci_read_config_dword(dev, ent_offset, &base_upper);
3311		ent_offset += 4;
3312
3313		flags |= IORESOURCE_MEM_64;
3314
3315		/* entry starts above 32-bit boundary, can't use */
3316		if (!support_64 && base_upper)
3317			goto out;
3318
3319		if (support_64)
3320			start |= ((u64)base_upper << 32);
3321	}
3322
3323	end = start + (max_offset | 0x03);
3324
3325	/* Read MaxOffset MSBs (if 64-bit entry) */
3326	if (max_offset & PCI_EA_IS_64) {
3327		u32 max_offset_upper;
3328
3329		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
3330		ent_offset += 4;
3331
3332		flags |= IORESOURCE_MEM_64;
3333
3334		/* entry too big, can't use */
3335		if (!support_64 && max_offset_upper)
3336			goto out;
3337
3338		if (support_64)
3339			end += ((u64)max_offset_upper << 32);
3340	}
3341
3342	if (end < start) {
3343		pci_err(dev, "EA Entry crosses address boundary\n");
3344		goto out;
3345	}
3346
3347	if (ent_size != ent_offset - offset) {
3348		pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
3349			ent_size, ent_offset - offset);
3350		goto out;
3351	}
3352
3353	res->name = pci_name(dev);
3354	res->start = start;
3355	res->end = end;
3356	res->flags = flags;
3357
3358	if (bei <= PCI_EA_BEI_BAR5)
3359		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3360			 res_name, res, prop);
3361	else if (bei == PCI_EA_BEI_ROM)
3362		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3363			 res_name, res, prop);
3364	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
3365		pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3366			 res_name, res, prop);
3367	else
3368		pci_info(dev, "BEI %d %pR: from Enhanced Allocation, properties %#02x\n",
3369			   bei, res, prop);
3370
3371out:
3372	return offset + ent_size;
3373}
3374
3375/* Enhanced Allocation Initialization */
3376void pci_ea_init(struct pci_dev *dev)
3377{
3378	int ea;
3379	u8 num_ent;
3380	int offset;
3381	int i;
3382
3383	/* find PCI EA capability in list */
3384	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
3385	if (!ea)
3386		return;
3387
3388	/* determine the number of entries */
3389	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
3390					&num_ent);
3391	num_ent &= PCI_EA_NUM_ENT_MASK;
3392
3393	offset = ea + PCI_EA_FIRST_ENT;
3394
3395	/* Skip DWORD 2 for type 1 functions */
3396	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
3397		offset += 4;
3398
3399	/* parse each EA entry */
3400	for (i = 0; i < num_ent; ++i)
3401		offset = pci_ea_read(dev, offset);
3402}
3403
3404static void pci_add_saved_cap(struct pci_dev *pci_dev,
3405	struct pci_cap_saved_state *new_cap)
3406{
3407	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
3408}
3409
3410/**
3411 * _pci_add_cap_save_buffer - allocate buffer for saving given
3412 *			      capability registers
3413 * @dev: the PCI device
3414 * @cap: the capability to allocate the buffer for
3415 * @extended: Standard or Extended capability ID
3416 * @size: requested size of the buffer
3417 */
3418static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
3419				    bool extended, unsigned int size)
3420{
3421	int pos;
3422	struct pci_cap_saved_state *save_state;
3423
3424	if (extended)
3425		pos = pci_find_ext_capability(dev, cap);
3426	else
3427		pos = pci_find_capability(dev, cap);
3428
3429	if (!pos)
3430		return 0;
3431
3432	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
3433	if (!save_state)
3434		return -ENOMEM;
3435
3436	save_state->cap.cap_nr = cap;
3437	save_state->cap.cap_extended = extended;
3438	save_state->cap.size = size;
3439	pci_add_saved_cap(dev, save_state);
3440
3441	return 0;
3442}
3443
3444int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3445{
3446	return _pci_add_cap_save_buffer(dev, cap, false, size);
3447}
3448
3449int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3450{
3451	return _pci_add_cap_save_buffer(dev, cap, true, size);
3452}
3453
3454/**
3455 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3456 * @dev: the PCI device
3457 */
3458void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3459{
3460	int error;
3461
3462	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3463					PCI_EXP_SAVE_REGS * sizeof(u16));
3464	if (error)
3465		pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3466
3467	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
3468	if (error)
3469		pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3470
3471	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
3472					    2 * sizeof(u16));
3473	if (error)
3474		pci_err(dev, "unable to allocate suspend buffer for LTR\n");
3475
3476	pci_allocate_vc_save_buffers(dev);
3477}
3478
3479void pci_free_cap_save_buffers(struct pci_dev *dev)
3480{
3481	struct pci_cap_saved_state *tmp;
3482	struct hlist_node *n;
3483
3484	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3485		kfree(tmp);
3486}
3487
3488/**
3489 * pci_configure_ari - enable or disable ARI forwarding
3490 * @dev: the PCI device
3491 *
3492 * If @dev and its upstream bridge both support ARI, enable ARI in the
3493 * bridge.  Otherwise, disable ARI in the bridge.
3494 */
3495void pci_configure_ari(struct pci_dev *dev)
3496{
3497	u32 cap;
3498	struct pci_dev *bridge;
3499
3500	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3501		return;
3502
3503	bridge = dev->bus->self;
3504	if (!bridge)
3505		return;
3506
3507	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3508	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3509		return;
3510
3511	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3512		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
3513					 PCI_EXP_DEVCTL2_ARI);
3514		bridge->ari_enabled = 1;
3515	} else {
3516		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
3517					   PCI_EXP_DEVCTL2_ARI);
3518		bridge->ari_enabled = 0;
3519	}
3520}
3521
3522static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3523{
3524	int pos;
3525	u16 cap, ctrl;
3526
3527	pos = pdev->acs_cap;
3528	if (!pos)
3529		return false;
3530
3531	/*
3532	 * Except for egress control, capabilities are either required
3533	 * or only required if controllable.  Features missing from the
3534	 * capability field can therefore be assumed as hard-wired enabled.
3535	 */
3536	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
3537	acs_flags &= (cap | PCI_ACS_EC);
3538
3539	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
3540	return (ctrl & acs_flags) == acs_flags;
3541}
3542
3543/**
3544 * pci_acs_enabled - test ACS against required flags for a given device
3545 * @pdev: device to test
3546 * @acs_flags: required PCI ACS flags
3547 *
3548 * Return true if the device supports the provided flags.  Automatically
3549 * filters out flags that are not implemented on multifunction devices.
3550 *
3551 * Note that this interface checks the effective ACS capabilities of the
3552 * device rather than the actual capabilities.  For instance, most single
3553 * function endpoints are not required to support ACS because they have no
3554 * opportunity for peer-to-peer access.  We therefore return 'true'
3555 * regardless of whether the device exposes an ACS capability.  This makes
3556 * it much easier for callers of this function to ignore the actual type
3557 * or topology of the device when testing ACS support.
3558 */
3559bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3560{
3561	int ret;
3562
3563	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
3564	if (ret >= 0)
3565		return ret > 0;
3566
3567	/*
3568	 * Conventional PCI and PCI-X devices never support ACS, either
3569	 * effectively or actually.  The shared bus topology implies that
3570	 * any device on the bus can receive or snoop DMA.
3571	 */
3572	if (!pci_is_pcie(pdev))
3573		return false;
3574
3575	switch (pci_pcie_type(pdev)) {
3576	/*
3577	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3578	 * but since their primary interface is PCI/X, we conservatively
3579	 * handle them as we would a non-PCIe device.
3580	 */
3581	case PCI_EXP_TYPE_PCIE_BRIDGE:
3582	/*
3583	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
3584	 * applicable... must never implement an ACS Extended Capability...".
3585	 * This seems arbitrary, but we take a conservative interpretation
3586	 * of this statement.
3587	 */
3588	case PCI_EXP_TYPE_PCI_BRIDGE:
3589	case PCI_EXP_TYPE_RC_EC:
3590		return false;
3591	/*
3592	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3593	 * implement ACS in order to indicate their peer-to-peer capabilities,
3594	 * regardless of whether they are single- or multi-function devices.
3595	 */
3596	case PCI_EXP_TYPE_DOWNSTREAM:
3597	case PCI_EXP_TYPE_ROOT_PORT:
3598		return pci_acs_flags_enabled(pdev, acs_flags);
3599	/*
3600	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3601	 * implemented by the remaining PCIe types to indicate peer-to-peer
3602	 * capabilities, but only when they are part of a multifunction
3603	 * device.  The footnote for section 6.12 indicates the specific
3604	 * PCIe types included here.
3605	 */
3606	case PCI_EXP_TYPE_ENDPOINT:
3607	case PCI_EXP_TYPE_UPSTREAM:
3608	case PCI_EXP_TYPE_LEG_END:
3609	case PCI_EXP_TYPE_RC_END:
3610		if (!pdev->multifunction)
3611			break;
3612
3613		return pci_acs_flags_enabled(pdev, acs_flags);
3614	}
3615
3616	/*
3617	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3618	 * to single function devices with the exception of downstream ports.
3619	 */
3620	return true;
3621}
3622
3623/**
3624 * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy
3625 * @start: starting downstream device
3626 * @end: ending upstream device or NULL to search to the root bus
3627 * @acs_flags: required flags
3628 *
3629 * Walk up a device tree from start to end testing PCI ACS support.  If
3630 * any step along the way does not support the required flags, return false.
3631 */
3632bool pci_acs_path_enabled(struct pci_dev *start,
3633			  struct pci_dev *end, u16 acs_flags)
3634{
3635	struct pci_dev *pdev, *parent = start;
3636
3637	do {
3638		pdev = parent;
3639
3640		if (!pci_acs_enabled(pdev, acs_flags))
3641			return false;
3642
3643		if (pci_is_root_bus(pdev->bus))
3644			return (end == NULL);
3645
3646		parent = pdev->bus->self;
3647	} while (pdev != end);
3648
3649	return true;
3650}
3651
3652/**
3653 * pci_acs_init - Initialize ACS if hardware supports it
3654 * @dev: the PCI device
3655 */
3656void pci_acs_init(struct pci_dev *dev)
3657{
3658	dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3659
3660	/*
3661	 * Attempt to enable ACS regardless of capability because some Root
3662	 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
3663	 * the standard ACS capability but still support ACS via those
3664	 * quirks.
3665	 */
3666	pci_enable_acs(dev);
3667}
3668
3669/**
3670 * pci_rebar_find_pos - find position of resize ctrl reg for BAR
3671 * @pdev: PCI device
3672 * @bar: BAR to find
3673 *
3674 * Helper to find the position of the ctrl register for a BAR.
3675 * Returns -ENOTSUPP if resizable BARs are not supported at all.
3676 * Returns -ENOENT if no ctrl register for the BAR could be found.
3677 */
3678static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3679{
3680	unsigned int pos, nbars, i;
3681	u32 ctrl;
3682
3683	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3684	if (!pos)
3685		return -ENOTSUPP;
3686
3687	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3688	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
 
3689
3690	for (i = 0; i < nbars; i++, pos += 8) {
3691		int bar_idx;
3692
3693		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3694		bar_idx = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, ctrl);
3695		if (bar_idx == bar)
3696			return pos;
3697	}
3698
3699	return -ENOENT;
3700}
3701
3702/**
3703 * pci_rebar_get_possible_sizes - get possible sizes for BAR
3704 * @pdev: PCI device
3705 * @bar: BAR to query
3706 *
3707 * Get the possible sizes of a resizable BAR as bitmask defined in the spec
3708 * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3709 */
3710u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3711{
3712	int pos;
3713	u32 cap;
3714
3715	pos = pci_rebar_find_pos(pdev, bar);
3716	if (pos < 0)
3717		return 0;
3718
3719	pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap);
3720	cap = FIELD_GET(PCI_REBAR_CAP_SIZES, cap);
3721
3722	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3723	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3724	    bar == 0 && cap == 0x700)
3725		return 0x3f00;
3726
3727	return cap;
3728}
3729EXPORT_SYMBOL(pci_rebar_get_possible_sizes);
3730
3731/**
3732 * pci_rebar_get_current_size - get the current size of a BAR
3733 * @pdev: PCI device
3734 * @bar: BAR to set size to
3735 *
3736 * Read the size of a BAR from the resizable BAR config.
3737 * Returns size if found or negative error code.
3738 */
3739int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3740{
3741	int pos;
3742	u32 ctrl;
3743
3744	pos = pci_rebar_find_pos(pdev, bar);
3745	if (pos < 0)
3746		return pos;
3747
3748	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3749	return FIELD_GET(PCI_REBAR_CTRL_BAR_SIZE, ctrl);
3750}
3751
3752/**
3753 * pci_rebar_set_size - set a new size for a BAR
3754 * @pdev: PCI device
3755 * @bar: BAR to set size to
3756 * @size: new size as defined in the spec (0=1MB, 19=512GB)
3757 *
3758 * Set the new size of a BAR as defined in the spec.
3759 * Returns zero if resizing was successful, error code otherwise.
3760 */
3761int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3762{
3763	int pos;
3764	u32 ctrl;
3765
3766	pos = pci_rebar_find_pos(pdev, bar);
3767	if (pos < 0)
3768		return pos;
3769
3770	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3771	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3772	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
3773	pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
3774	return 0;
3775}
3776
3777/**
3778 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3779 * @dev: the PCI device
3780 * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3781 *	PCI_EXP_DEVCAP2_ATOMIC_COMP32
3782 *	PCI_EXP_DEVCAP2_ATOMIC_COMP64
3783 *	PCI_EXP_DEVCAP2_ATOMIC_COMP128
3784 *
3785 * Return 0 if all upstream bridges support AtomicOp routing, egress
3786 * blocking is disabled on all upstream ports, and the root port supports
3787 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3788 * AtomicOp completion), or negative otherwise.
3789 */
3790int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3791{
3792	struct pci_bus *bus = dev->bus;
3793	struct pci_dev *bridge;
3794	u32 cap, ctl2;
3795
3796	/*
3797	 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
3798	 * in Device Control 2 is reserved in VFs and the PF value applies
3799	 * to all associated VFs.
3800	 */
3801	if (dev->is_virtfn)
3802		return -EINVAL;
3803
3804	if (!pci_is_pcie(dev))
3805		return -EINVAL;
3806
3807	/*
3808	 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3809	 * AtomicOp requesters.  For now, we only support endpoints as
3810	 * requesters and root ports as completers.  No endpoints as
3811	 * completers, and no peer-to-peer.
3812	 */
3813
3814	switch (pci_pcie_type(dev)) {
3815	case PCI_EXP_TYPE_ENDPOINT:
3816	case PCI_EXP_TYPE_LEG_END:
3817	case PCI_EXP_TYPE_RC_END:
3818		break;
3819	default:
3820		return -EINVAL;
3821	}
3822
3823	while (bus->parent) {
3824		bridge = bus->self;
3825
3826		pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3827
3828		switch (pci_pcie_type(bridge)) {
3829		/* Ensure switch ports support AtomicOp routing */
3830		case PCI_EXP_TYPE_UPSTREAM:
3831		case PCI_EXP_TYPE_DOWNSTREAM:
3832			if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3833				return -EINVAL;
3834			break;
3835
3836		/* Ensure root port supports all the sizes we care about */
3837		case PCI_EXP_TYPE_ROOT_PORT:
3838			if ((cap & cap_mask) != cap_mask)
3839				return -EINVAL;
3840			break;
3841		}
3842
3843		/* Ensure upstream ports don't block AtomicOps on egress */
3844		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) {
3845			pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2,
3846						   &ctl2);
3847			if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3848				return -EINVAL;
3849		}
3850
3851		bus = bus->parent;
3852	}
3853
3854	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3855				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
3856	return 0;
3857}
3858EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3859
3860/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3861 * pci_release_region - Release a PCI bar
3862 * @pdev: PCI device whose resources were previously reserved by
3863 *	  pci_request_region()
3864 * @bar: BAR to release
3865 *
3866 * Releases the PCI I/O and memory resources previously reserved by a
3867 * successful call to pci_request_region().  Call this function only
3868 * after all use of the PCI regions has ceased.
3869 */
3870void pci_release_region(struct pci_dev *pdev, int bar)
3871{
3872	struct pci_devres *dr;
3873
3874	if (pci_resource_len(pdev, bar) == 0)
3875		return;
3876	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3877		release_region(pci_resource_start(pdev, bar),
3878				pci_resource_len(pdev, bar));
3879	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3880		release_mem_region(pci_resource_start(pdev, bar),
3881				pci_resource_len(pdev, bar));
3882
3883	dr = find_pci_dr(pdev);
3884	if (dr)
3885		dr->region_mask &= ~(1 << bar);
3886}
3887EXPORT_SYMBOL(pci_release_region);
3888
3889/**
3890 * __pci_request_region - Reserved PCI I/O and memory resource
3891 * @pdev: PCI device whose resources are to be reserved
3892 * @bar: BAR to be reserved
3893 * @res_name: Name to be associated with resource.
3894 * @exclusive: whether the region access is exclusive or not
3895 *
3896 * Mark the PCI region associated with PCI device @pdev BAR @bar as
3897 * being reserved by owner @res_name.  Do not access any
3898 * address inside the PCI regions unless this call returns
3899 * successfully.
3900 *
3901 * If @exclusive is set, then the region is marked so that userspace
3902 * is explicitly not allowed to map the resource via /dev/mem or
3903 * sysfs MMIO access.
3904 *
3905 * Returns 0 on success, or %EBUSY on error.  A warning
3906 * message is also printed on failure.
3907 */
3908static int __pci_request_region(struct pci_dev *pdev, int bar,
3909				const char *res_name, int exclusive)
3910{
3911	struct pci_devres *dr;
3912
3913	if (pci_resource_len(pdev, bar) == 0)
3914		return 0;
3915
3916	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3917		if (!request_region(pci_resource_start(pdev, bar),
3918			    pci_resource_len(pdev, bar), res_name))
3919			goto err_out;
3920	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3921		if (!__request_mem_region(pci_resource_start(pdev, bar),
3922					pci_resource_len(pdev, bar), res_name,
3923					exclusive))
3924			goto err_out;
3925	}
3926
3927	dr = find_pci_dr(pdev);
3928	if (dr)
3929		dr->region_mask |= 1 << bar;
3930
3931	return 0;
3932
3933err_out:
3934	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
3935		 &pdev->resource[bar]);
3936	return -EBUSY;
3937}
3938
3939/**
3940 * pci_request_region - Reserve PCI I/O and memory resource
3941 * @pdev: PCI device whose resources are to be reserved
3942 * @bar: BAR to be reserved
3943 * @res_name: Name to be associated with resource
3944 *
3945 * Mark the PCI region associated with PCI device @pdev BAR @bar as
3946 * being reserved by owner @res_name.  Do not access any
3947 * address inside the PCI regions unless this call returns
3948 * successfully.
3949 *
3950 * Returns 0 on success, or %EBUSY on error.  A warning
3951 * message is also printed on failure.
3952 */
3953int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
3954{
3955	return __pci_request_region(pdev, bar, res_name, 0);
3956}
3957EXPORT_SYMBOL(pci_request_region);
3958
3959/**
3960 * pci_release_selected_regions - Release selected PCI I/O and memory resources
3961 * @pdev: PCI device whose resources were previously reserved
3962 * @bars: Bitmask of BARs to be released
3963 *
3964 * Release selected PCI I/O and memory resources previously reserved.
3965 * Call this function only after all use of the PCI regions has ceased.
3966 */
3967void pci_release_selected_regions(struct pci_dev *pdev, int bars)
3968{
3969	int i;
3970
3971	for (i = 0; i < PCI_STD_NUM_BARS; i++)
3972		if (bars & (1 << i))
3973			pci_release_region(pdev, i);
3974}
3975EXPORT_SYMBOL(pci_release_selected_regions);
3976
3977static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
3978					  const char *res_name, int excl)
3979{
3980	int i;
3981
3982	for (i = 0; i < PCI_STD_NUM_BARS; i++)
3983		if (bars & (1 << i))
3984			if (__pci_request_region(pdev, i, res_name, excl))
3985				goto err_out;
3986	return 0;
3987
3988err_out:
3989	while (--i >= 0)
3990		if (bars & (1 << i))
3991			pci_release_region(pdev, i);
3992
3993	return -EBUSY;
3994}
3995
3996
3997/**
3998 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
3999 * @pdev: PCI device whose resources are to be reserved
4000 * @bars: Bitmask of BARs to be requested
4001 * @res_name: Name to be associated with resource
4002 */
4003int pci_request_selected_regions(struct pci_dev *pdev, int bars,
4004				 const char *res_name)
4005{
4006	return __pci_request_selected_regions(pdev, bars, res_name, 0);
4007}
4008EXPORT_SYMBOL(pci_request_selected_regions);
4009
4010int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
4011					   const char *res_name)
4012{
4013	return __pci_request_selected_regions(pdev, bars, res_name,
4014			IORESOURCE_EXCLUSIVE);
4015}
4016EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
4017
4018/**
4019 * pci_release_regions - Release reserved PCI I/O and memory resources
4020 * @pdev: PCI device whose resources were previously reserved by
4021 *	  pci_request_regions()
4022 *
4023 * Releases all PCI I/O and memory resources previously reserved by a
4024 * successful call to pci_request_regions().  Call this function only
4025 * after all use of the PCI regions has ceased.
4026 */
4027
4028void pci_release_regions(struct pci_dev *pdev)
4029{
4030	pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
4031}
4032EXPORT_SYMBOL(pci_release_regions);
4033
4034/**
4035 * pci_request_regions - Reserve PCI I/O and memory resources
4036 * @pdev: PCI device whose resources are to be reserved
4037 * @res_name: Name to be associated with resource.
4038 *
4039 * Mark all PCI regions associated with PCI device @pdev as
4040 * being reserved by owner @res_name.  Do not access any
4041 * address inside the PCI regions unless this call returns
4042 * successfully.
4043 *
4044 * Returns 0 on success, or %EBUSY on error.  A warning
4045 * message is also printed on failure.
4046 */
4047int pci_request_regions(struct pci_dev *pdev, const char *res_name)
4048{
4049	return pci_request_selected_regions(pdev,
4050			((1 << PCI_STD_NUM_BARS) - 1), res_name);
4051}
4052EXPORT_SYMBOL(pci_request_regions);
4053
4054/**
4055 * pci_request_regions_exclusive - Reserve PCI I/O and memory resources
4056 * @pdev: PCI device whose resources are to be reserved
4057 * @res_name: Name to be associated with resource.
4058 *
4059 * Mark all PCI regions associated with PCI device @pdev as being reserved
4060 * by owner @res_name.  Do not access any address inside the PCI regions
4061 * unless this call returns successfully.
4062 *
4063 * pci_request_regions_exclusive() will mark the region so that /dev/mem
4064 * and the sysfs MMIO access will not be allowed.
4065 *
4066 * Returns 0 on success, or %EBUSY on error.  A warning message is also
4067 * printed on failure.
4068 */
4069int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
4070{
4071	return pci_request_selected_regions_exclusive(pdev,
4072				((1 << PCI_STD_NUM_BARS) - 1), res_name);
4073}
4074EXPORT_SYMBOL(pci_request_regions_exclusive);
4075
4076/*
4077 * Record the PCI IO range (expressed as CPU physical address + size).
4078 * Return a negative value if an error has occurred, zero otherwise
4079 */
4080int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
4081			resource_size_t	size)
4082{
4083	int ret = 0;
4084#ifdef PCI_IOBASE
4085	struct logic_pio_hwaddr *range;
4086
4087	if (!size || addr + size < addr)
4088		return -EINVAL;
4089
4090	range = kzalloc(sizeof(*range), GFP_ATOMIC);
4091	if (!range)
4092		return -ENOMEM;
4093
4094	range->fwnode = fwnode;
4095	range->size = size;
4096	range->hw_start = addr;
4097	range->flags = LOGIC_PIO_CPU_MMIO;
4098
4099	ret = logic_pio_register_range(range);
4100	if (ret)
4101		kfree(range);
4102
4103	/* Ignore duplicates due to deferred probing */
4104	if (ret == -EEXIST)
4105		ret = 0;
4106#endif
4107
4108	return ret;
4109}
4110
4111phys_addr_t pci_pio_to_address(unsigned long pio)
4112{
 
 
4113#ifdef PCI_IOBASE
4114	if (pio < MMIO_UPPER_LIMIT)
4115		return logic_pio_to_hwaddr(pio);
 
 
4116#endif
4117
4118	return (phys_addr_t) OF_BAD_ADDR;
4119}
4120EXPORT_SYMBOL_GPL(pci_pio_to_address);
4121
4122unsigned long __weak pci_address_to_pio(phys_addr_t address)
4123{
4124#ifdef PCI_IOBASE
4125	return logic_pio_trans_cpuaddr(address);
4126#else
4127	if (address > IO_SPACE_LIMIT)
4128		return (unsigned long)-1;
4129
4130	return (unsigned long) address;
4131#endif
4132}
4133
4134/**
4135 * pci_remap_iospace - Remap the memory mapped I/O space
4136 * @res: Resource describing the I/O space
4137 * @phys_addr: physical address of range to be mapped
4138 *
4139 * Remap the memory mapped I/O space described by the @res and the CPU
4140 * physical address @phys_addr into virtual address space.  Only
4141 * architectures that have memory mapped IO functions defined (and the
4142 * PCI_IOBASE value defined) should call this function.
4143 */
4144#ifndef pci_remap_iospace
4145int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
4146{
4147#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4148	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4149
4150	if (!(res->flags & IORESOURCE_IO))
4151		return -EINVAL;
4152
4153	if (res->end > IO_SPACE_LIMIT)
4154		return -EINVAL;
4155
4156	return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
4157			       pgprot_device(PAGE_KERNEL));
4158#else
4159	/*
4160	 * This architecture does not have memory mapped I/O space,
4161	 * so this function should never be called
4162	 */
4163	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
4164	return -ENODEV;
4165#endif
4166}
4167EXPORT_SYMBOL(pci_remap_iospace);
4168#endif
4169
4170/**
4171 * pci_unmap_iospace - Unmap the memory mapped I/O space
4172 * @res: resource to be unmapped
4173 *
4174 * Unmap the CPU virtual address @res from virtual address space.  Only
4175 * architectures that have memory mapped IO functions defined (and the
4176 * PCI_IOBASE value defined) should call this function.
4177 */
4178void pci_unmap_iospace(struct resource *res)
4179{
4180#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4181	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4182
4183	vunmap_range(vaddr, vaddr + resource_size(res));
4184#endif
4185}
4186EXPORT_SYMBOL(pci_unmap_iospace);
4187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4188static void __pci_set_master(struct pci_dev *dev, bool enable)
4189{
4190	u16 old_cmd, cmd;
4191
4192	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
4193	if (enable)
4194		cmd = old_cmd | PCI_COMMAND_MASTER;
4195	else
4196		cmd = old_cmd & ~PCI_COMMAND_MASTER;
4197	if (cmd != old_cmd) {
4198		pci_dbg(dev, "%s bus mastering\n",
4199			enable ? "enabling" : "disabling");
4200		pci_write_config_word(dev, PCI_COMMAND, cmd);
4201	}
4202	dev->is_busmaster = enable;
4203}
4204
4205/**
4206 * pcibios_setup - process "pci=" kernel boot arguments
4207 * @str: string used to pass in "pci=" kernel boot arguments
4208 *
4209 * Process kernel boot arguments.  This is the default implementation.
4210 * Architecture specific implementations can override this as necessary.
4211 */
4212char * __weak __init pcibios_setup(char *str)
4213{
4214	return str;
4215}
4216
4217/**
4218 * pcibios_set_master - enable PCI bus-mastering for device dev
4219 * @dev: the PCI device to enable
4220 *
4221 * Enables PCI bus-mastering for the device.  This is the default
4222 * implementation.  Architecture specific implementations can override
4223 * this if necessary.
4224 */
4225void __weak pcibios_set_master(struct pci_dev *dev)
4226{
4227	u8 lat;
4228
4229	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4230	if (pci_is_pcie(dev))
4231		return;
4232
4233	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
4234	if (lat < 16)
4235		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4236	else if (lat > pcibios_max_latency)
4237		lat = pcibios_max_latency;
4238	else
4239		return;
4240
4241	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
4242}
4243
4244/**
4245 * pci_set_master - enables bus-mastering for device dev
4246 * @dev: the PCI device to enable
4247 *
4248 * Enables bus-mastering on the device and calls pcibios_set_master()
4249 * to do the needed arch specific settings.
4250 */
4251void pci_set_master(struct pci_dev *dev)
4252{
4253	__pci_set_master(dev, true);
4254	pcibios_set_master(dev);
4255}
4256EXPORT_SYMBOL(pci_set_master);
4257
4258/**
4259 * pci_clear_master - disables bus-mastering for device dev
4260 * @dev: the PCI device to disable
4261 */
4262void pci_clear_master(struct pci_dev *dev)
4263{
4264	__pci_set_master(dev, false);
4265}
4266EXPORT_SYMBOL(pci_clear_master);
4267
4268/**
4269 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4270 * @dev: the PCI device for which MWI is to be enabled
4271 *
4272 * Helper function for pci_set_mwi.
4273 * Originally copied from drivers/net/acenic.c.
4274 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4275 *
4276 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4277 */
4278int pci_set_cacheline_size(struct pci_dev *dev)
4279{
4280	u8 cacheline_size;
4281
4282	if (!pci_cache_line_size)
4283		return -EINVAL;
4284
4285	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4286	   equal to or multiple of the right value. */
4287	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4288	if (cacheline_size >= pci_cache_line_size &&
4289	    (cacheline_size % pci_cache_line_size) == 0)
4290		return 0;
4291
4292	/* Write the correct value. */
4293	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
4294	/* Read it back. */
4295	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4296	if (cacheline_size == pci_cache_line_size)
4297		return 0;
4298
4299	pci_dbg(dev, "cache line size of %d is not supported\n",
4300		   pci_cache_line_size << 2);
4301
4302	return -EINVAL;
4303}
4304EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4305
4306/**
4307 * pci_set_mwi - enables memory-write-invalidate PCI transaction
4308 * @dev: the PCI device for which MWI is enabled
4309 *
4310 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4311 *
4312 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4313 */
4314int pci_set_mwi(struct pci_dev *dev)
4315{
4316#ifdef PCI_DISABLE_MWI
4317	return 0;
4318#else
4319	int rc;
4320	u16 cmd;
4321
4322	rc = pci_set_cacheline_size(dev);
4323	if (rc)
4324		return rc;
4325
4326	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4327	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4328		pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4329		cmd |= PCI_COMMAND_INVALIDATE;
4330		pci_write_config_word(dev, PCI_COMMAND, cmd);
4331	}
4332	return 0;
4333#endif
4334}
4335EXPORT_SYMBOL(pci_set_mwi);
4336
4337/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4338 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4339 * @dev: the PCI device for which MWI is enabled
4340 *
4341 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4342 * Callers are not required to check the return value.
4343 *
4344 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4345 */
4346int pci_try_set_mwi(struct pci_dev *dev)
4347{
4348#ifdef PCI_DISABLE_MWI
4349	return 0;
4350#else
4351	return pci_set_mwi(dev);
4352#endif
4353}
4354EXPORT_SYMBOL(pci_try_set_mwi);
4355
4356/**
4357 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4358 * @dev: the PCI device to disable
4359 *
4360 * Disables PCI Memory-Write-Invalidate transaction on the device
4361 */
4362void pci_clear_mwi(struct pci_dev *dev)
4363{
4364#ifndef PCI_DISABLE_MWI
4365	u16 cmd;
4366
4367	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4368	if (cmd & PCI_COMMAND_INVALIDATE) {
4369		cmd &= ~PCI_COMMAND_INVALIDATE;
4370		pci_write_config_word(dev, PCI_COMMAND, cmd);
4371	}
4372#endif
4373}
4374EXPORT_SYMBOL(pci_clear_mwi);
4375
4376/**
4377 * pci_disable_parity - disable parity checking for device
4378 * @dev: the PCI device to operate on
4379 *
4380 * Disable parity checking for device @dev
4381 */
4382void pci_disable_parity(struct pci_dev *dev)
4383{
4384	u16 cmd;
4385
4386	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4387	if (cmd & PCI_COMMAND_PARITY) {
4388		cmd &= ~PCI_COMMAND_PARITY;
4389		pci_write_config_word(dev, PCI_COMMAND, cmd);
4390	}
4391}
4392
4393/**
4394 * pci_intx - enables/disables PCI INTx for device dev
4395 * @pdev: the PCI device to operate on
4396 * @enable: boolean: whether to enable or disable PCI INTx
4397 *
4398 * Enables/disables PCI INTx for device @pdev
4399 */
4400void pci_intx(struct pci_dev *pdev, int enable)
4401{
4402	u16 pci_command, new;
4403
4404	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
4405
4406	if (enable)
4407		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4408	else
4409		new = pci_command | PCI_COMMAND_INTX_DISABLE;
4410
4411	if (new != pci_command) {
4412		struct pci_devres *dr;
4413
4414		pci_write_config_word(pdev, PCI_COMMAND, new);
4415
4416		dr = find_pci_dr(pdev);
4417		if (dr && !dr->restore_intx) {
4418			dr->restore_intx = 1;
4419			dr->orig_intx = !enable;
4420		}
4421	}
4422}
4423EXPORT_SYMBOL_GPL(pci_intx);
4424
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4425/**
4426 * pci_wait_for_pending_transaction - wait for pending transaction
4427 * @dev: the PCI device to operate on
4428 *
4429 * Return 0 if transaction is pending 1 otherwise.
4430 */
4431int pci_wait_for_pending_transaction(struct pci_dev *dev)
4432{
4433	if (!pci_is_pcie(dev))
4434		return 1;
4435
4436	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4437				    PCI_EXP_DEVSTA_TRPND);
4438}
4439EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4440
4441/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4442 * pcie_flr - initiate a PCIe function level reset
4443 * @dev: device to reset
4444 *
4445 * Initiate a function level reset unconditionally on @dev without
4446 * checking any flags and DEVCAP
 
4447 */
4448int pcie_flr(struct pci_dev *dev)
4449{
4450	if (!pci_wait_for_pending_transaction(dev))
4451		pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4452
4453	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4454
4455	if (dev->imm_ready)
4456		return 0;
4457
4458	/*
4459	 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4460	 * 100ms, but may silently discard requests while the FLR is in
4461	 * progress.  Wait 100ms before trying to access the device.
4462	 */
4463	msleep(100);
4464
4465	return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4466}
4467EXPORT_SYMBOL_GPL(pcie_flr);
4468
4469/**
4470 * pcie_reset_flr - initiate a PCIe function level reset
4471 * @dev: device to reset
4472 * @probe: if true, return 0 if device can be reset this way
4473 *
4474 * Initiate a function level reset on @dev.
4475 */
4476int pcie_reset_flr(struct pci_dev *dev, bool probe)
4477{
4478	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4479		return -ENOTTY;
4480
4481	if (!(dev->devcap & PCI_EXP_DEVCAP_FLR))
4482		return -ENOTTY;
4483
4484	if (probe)
4485		return 0;
4486
4487	return pcie_flr(dev);
4488}
4489EXPORT_SYMBOL_GPL(pcie_reset_flr);
4490
4491static int pci_af_flr(struct pci_dev *dev, bool probe)
4492{
4493	int pos;
4494	u8 cap;
4495
4496	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4497	if (!pos)
4498		return -ENOTTY;
4499
4500	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4501		return -ENOTTY;
4502
4503	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
4504	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4505		return -ENOTTY;
4506
4507	if (probe)
4508		return 0;
4509
4510	/*
4511	 * Wait for Transaction Pending bit to clear.  A word-aligned test
4512	 * is used, so we use the control offset rather than status and shift
4513	 * the test bit to match.
4514	 */
4515	if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
4516				 PCI_AF_STATUS_TP << 8))
4517		pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4518
4519	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4520
4521	if (dev->imm_ready)
4522		return 0;
4523
4524	/*
4525	 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4526	 * updated 27 July 2006; a device must complete an FLR within
4527	 * 100ms, but may silently discard requests while the FLR is in
4528	 * progress.  Wait 100ms before trying to access the device.
4529	 */
4530	msleep(100);
4531
4532	return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4533}
4534
4535/**
4536 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4537 * @dev: Device to reset.
4538 * @probe: if true, return 0 if the device can be reset this way.
4539 *
4540 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4541 * unset, it will be reinitialized internally when going from PCI_D3hot to
4542 * PCI_D0.  If that's the case and the device is not in a low-power state
4543 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4544 *
4545 * NOTE: This causes the caller to sleep for twice the device power transition
4546 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4547 * by default (i.e. unless the @dev's d3hot_delay field has a different value).
4548 * Moreover, only devices in D0 can be reset by this function.
4549 */
4550static int pci_pm_reset(struct pci_dev *dev, bool probe)
4551{
4552	u16 csr;
4553
4554	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4555		return -ENOTTY;
4556
4557	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
4558	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4559		return -ENOTTY;
4560
4561	if (probe)
4562		return 0;
4563
4564	if (dev->current_state != PCI_D0)
4565		return -EINVAL;
4566
4567	csr &= ~PCI_PM_CTRL_STATE_MASK;
4568	csr |= PCI_D3hot;
4569	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4570	pci_dev_d3_sleep(dev);
4571
4572	csr &= ~PCI_PM_CTRL_STATE_MASK;
4573	csr |= PCI_D0;
4574	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4575	pci_dev_d3_sleep(dev);
4576
4577	return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4578}
4579
4580/**
4581 * pcie_wait_for_link_status - Wait for link status change
4582 * @pdev: Device whose link to wait for.
4583 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE.
4584 * @active: Waiting for active or inactive?
4585 *
4586 * Return 0 if successful, or -ETIMEDOUT if status has not changed within
4587 * PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4588 */
4589static int pcie_wait_for_link_status(struct pci_dev *pdev,
4590				     bool use_lt, bool active)
4591{
4592	u16 lnksta_mask, lnksta_match;
4593	unsigned long end_jiffies;
4594	u16 lnksta;
4595
4596	lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA;
4597	lnksta_match = active ? lnksta_mask : 0;
4598
4599	end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS);
4600	do {
4601		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
4602		if ((lnksta & lnksta_mask) == lnksta_match)
4603			return 0;
4604		msleep(1);
4605	} while (time_before(jiffies, end_jiffies));
4606
4607	return -ETIMEDOUT;
4608}
4609
4610/**
4611 * pcie_retrain_link - Request a link retrain and wait for it to complete
4612 * @pdev: Device whose link to retrain.
4613 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status.
4614 *
4615 * Retrain completion status is retrieved from the Link Status Register
4616 * according to @use_lt.  It is not verified whether the use of the DLLLA
4617 * bit is valid.
4618 *
4619 * Return 0 if successful, or -ETIMEDOUT if training has not completed
4620 * within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4621 */
4622int pcie_retrain_link(struct pci_dev *pdev, bool use_lt)
4623{
4624	int rc;
4625
4626	/*
4627	 * Ensure the updated LNKCTL parameters are used during link
4628	 * training by checking that there is no ongoing link training to
4629	 * avoid LTSSM race as recommended in Implementation Note at the
4630	 * end of PCIe r6.0.1 sec 7.5.3.7.
4631	 */
4632	rc = pcie_wait_for_link_status(pdev, true, false);
4633	if (rc)
4634		return rc;
4635
4636	pcie_capability_set_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4637	if (pdev->clear_retrain_link) {
4638		/*
4639		 * Due to an erratum in some devices the Retrain Link bit
4640		 * needs to be cleared again manually to allow the link
4641		 * training to succeed.
4642		 */
4643		pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4644	}
4645
4646	return pcie_wait_for_link_status(pdev, use_lt, !use_lt);
4647}
4648
4649/**
4650 * pcie_wait_for_link_delay - Wait until link is active or inactive
4651 * @pdev: Bridge device
4652 * @active: waiting for active or inactive?
4653 * @delay: Delay to wait after link has become active (in ms)
4654 *
4655 * Use this to wait till link becomes active or inactive.
4656 */
4657static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
4658				     int delay)
4659{
4660	int rc;
 
 
4661
4662	/*
4663	 * Some controllers might not implement link active reporting. In this
4664	 * case, we wait for 1000 ms + any delay requested by the caller.
4665	 */
4666	if (!pdev->link_active_reporting) {
4667		msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay);
4668		return true;
4669	}
4670
4671	/*
4672	 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
4673	 * after which we should expect an link active if the reset was
4674	 * successful. If so, software must wait a minimum 100ms before sending
4675	 * configuration requests to devices downstream this port.
4676	 *
4677	 * If the link fails to activate, either the device was physically
4678	 * removed or the link is permanently failed.
4679	 */
4680	if (active)
4681		msleep(20);
4682	rc = pcie_wait_for_link_status(pdev, false, active);
4683	if (active) {
4684		if (rc)
4685			rc = pcie_failed_link_retrain(pdev);
4686		if (rc)
4687			return false;
4688
4689		msleep(delay);
4690		return true;
4691	}
 
 
4692
4693	if (rc)
4694		return false;
4695
4696	return true;
4697}
4698
4699/**
4700 * pcie_wait_for_link - Wait until link is active or inactive
4701 * @pdev: Bridge device
4702 * @active: waiting for active or inactive?
4703 *
4704 * Use this to wait till link becomes active or inactive.
4705 */
4706bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
4707{
4708	return pcie_wait_for_link_delay(pdev, active, 100);
4709}
4710
4711/*
4712 * Find maximum D3cold delay required by all the devices on the bus.  The
4713 * spec says 100 ms, but firmware can lower it and we allow drivers to
4714 * increase it as well.
4715 *
4716 * Called with @pci_bus_sem locked for reading.
4717 */
4718static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
4719{
4720	const struct pci_dev *pdev;
4721	int min_delay = 100;
4722	int max_delay = 0;
4723
4724	list_for_each_entry(pdev, &bus->devices, bus_list) {
4725		if (pdev->d3cold_delay < min_delay)
4726			min_delay = pdev->d3cold_delay;
4727		if (pdev->d3cold_delay > max_delay)
4728			max_delay = pdev->d3cold_delay;
4729	}
4730
4731	return max(min_delay, max_delay);
4732}
4733
4734/**
4735 * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
4736 * @dev: PCI bridge
4737 * @reset_type: reset type in human-readable form
4738 *
4739 * Handle necessary delays before access to the devices on the secondary
4740 * side of the bridge are permitted after D3cold to D0 transition
4741 * or Conventional Reset.
4742 *
4743 * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
4744 * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
4745 * 4.3.2.
4746 *
4747 * Return 0 on success or -ENOTTY if the first device on the secondary bus
4748 * failed to become accessible.
4749 */
4750int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
4751{
4752	struct pci_dev *child;
4753	int delay;
4754
4755	if (pci_dev_is_disconnected(dev))
4756		return 0;
4757
4758	if (!pci_is_bridge(dev))
4759		return 0;
4760
4761	down_read(&pci_bus_sem);
4762
4763	/*
4764	 * We only deal with devices that are present currently on the bus.
4765	 * For any hot-added devices the access delay is handled in pciehp
4766	 * board_added(). In case of ACPI hotplug the firmware is expected
4767	 * to configure the devices before OS is notified.
4768	 */
4769	if (!dev->subordinate || list_empty(&dev->subordinate->devices)) {
4770		up_read(&pci_bus_sem);
4771		return 0;
4772	}
4773
4774	/* Take d3cold_delay requirements into account */
4775	delay = pci_bus_max_d3cold_delay(dev->subordinate);
4776	if (!delay) {
4777		up_read(&pci_bus_sem);
4778		return 0;
4779	}
4780
4781	child = list_first_entry(&dev->subordinate->devices, struct pci_dev,
4782				 bus_list);
4783	up_read(&pci_bus_sem);
4784
4785	/*
4786	 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
4787	 * accessing the device after reset (that is 1000 ms + 100 ms).
 
 
4788	 */
4789	if (!pci_is_pcie(dev)) {
4790		pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
4791		msleep(1000 + delay);
4792		return 0;
4793	}
4794
4795	/*
4796	 * For PCIe downstream and root ports that do not support speeds
4797	 * greater than 5 GT/s need to wait minimum 100 ms. For higher
4798	 * speeds (gen3) we need to wait first for the data link layer to
4799	 * become active.
4800	 *
4801	 * However, 100 ms is the minimum and the PCIe spec says the
4802	 * software must allow at least 1s before it can determine that the
4803	 * device that did not respond is a broken device. Also device can
4804	 * take longer than that to respond if it indicates so through Request
4805	 * Retry Status completions.
 
 
4806	 *
4807	 * Therefore we wait for 100 ms and check for the device presence
4808	 * until the timeout expires.
4809	 */
4810	if (!pcie_downstream_port(dev))
4811		return 0;
4812
4813	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
4814		u16 status;
4815
4816		pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
4817		msleep(delay);
4818
4819		if (!pci_dev_wait(child, reset_type, PCI_RESET_WAIT - delay))
4820			return 0;
4821
4822		/*
4823		 * If the port supports active link reporting we now check
4824		 * whether the link is active and if not bail out early with
4825		 * the assumption that the device is not present anymore.
4826		 */
4827		if (!dev->link_active_reporting)
4828			return -ENOTTY;
4829
4830		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &status);
4831		if (!(status & PCI_EXP_LNKSTA_DLLLA))
4832			return -ENOTTY;
4833
4834		return pci_dev_wait(child, reset_type,
4835				    PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT);
4836	}
4837
4838	pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
4839		delay);
4840	if (!pcie_wait_for_link_delay(dev, true, delay)) {
4841		/* Did not train, no need to wait any further */
4842		pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
4843		return -ENOTTY;
4844	}
4845
4846	return pci_dev_wait(child, reset_type,
4847			    PCIE_RESET_READY_POLL_MS - delay);
4848}
4849
4850void pci_reset_secondary_bus(struct pci_dev *dev)
4851{
4852	u16 ctrl;
4853
4854	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
4855	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
4856	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4857
4858	/*
4859	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
4860	 * this to 2ms to ensure that we meet the minimum requirement.
4861	 */
4862	msleep(2);
4863
4864	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
4865	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
 
 
 
 
 
 
 
 
 
4866}
4867
4868void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
4869{
4870	pci_reset_secondary_bus(dev);
4871}
4872
4873/**
4874 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
4875 * @dev: Bridge device
4876 *
4877 * Use the bridge control register to assert reset on the secondary bus.
4878 * Devices on the secondary bus are left in power-on state.
4879 */
4880int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
4881{
4882	pcibios_reset_secondary_bus(dev);
4883
4884	return pci_bridge_wait_for_secondary_bus(dev, "bus reset");
4885}
4886EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
4887
4888static int pci_parent_bus_reset(struct pci_dev *dev, bool probe)
4889{
4890	struct pci_dev *pdev;
4891
4892	if (pci_is_root_bus(dev->bus) || dev->subordinate ||
4893	    !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4894		return -ENOTTY;
4895
4896	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4897		if (pdev != dev)
4898			return -ENOTTY;
4899
4900	if (probe)
4901		return 0;
4902
4903	return pci_bridge_secondary_bus_reset(dev->bus->self);
4904}
4905
4906static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe)
4907{
4908	int rc = -ENOTTY;
4909
4910	if (!hotplug || !try_module_get(hotplug->owner))
4911		return rc;
4912
4913	if (hotplug->ops->reset_slot)
4914		rc = hotplug->ops->reset_slot(hotplug, probe);
4915
4916	module_put(hotplug->owner);
4917
4918	return rc;
4919}
4920
4921static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe)
4922{
4923	if (dev->multifunction || dev->subordinate || !dev->slot ||
4924	    dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4925		return -ENOTTY;
4926
4927	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
4928}
4929
4930static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
4931{
4932	int rc;
4933
4934	rc = pci_dev_reset_slot_function(dev, probe);
4935	if (rc != -ENOTTY)
4936		return rc;
4937	return pci_parent_bus_reset(dev, probe);
4938}
4939
4940void pci_dev_lock(struct pci_dev *dev)
4941{
 
4942	/* block PM suspend, driver probe, etc. */
4943	device_lock(&dev->dev);
4944	pci_cfg_access_lock(dev);
4945}
4946EXPORT_SYMBOL_GPL(pci_dev_lock);
4947
4948/* Return 1 on successful lock, 0 on contention */
4949int pci_dev_trylock(struct pci_dev *dev)
4950{
4951	if (device_trylock(&dev->dev)) {
4952		if (pci_cfg_access_trylock(dev))
4953			return 1;
4954		device_unlock(&dev->dev);
4955	}
4956
4957	return 0;
4958}
4959EXPORT_SYMBOL_GPL(pci_dev_trylock);
4960
4961void pci_dev_unlock(struct pci_dev *dev)
4962{
4963	pci_cfg_access_unlock(dev);
4964	device_unlock(&dev->dev);
 
4965}
4966EXPORT_SYMBOL_GPL(pci_dev_unlock);
4967
4968static void pci_dev_save_and_disable(struct pci_dev *dev)
4969{
4970	const struct pci_error_handlers *err_handler =
4971			dev->driver ? dev->driver->err_handler : NULL;
4972
4973	/*
4974	 * dev->driver->err_handler->reset_prepare() is protected against
4975	 * races with ->remove() by the device lock, which must be held by
4976	 * the caller.
4977	 */
4978	if (err_handler && err_handler->reset_prepare)
4979		err_handler->reset_prepare(dev);
4980
4981	/*
4982	 * Wake-up device prior to save.  PM registers default to D0 after
4983	 * reset and a simple register restore doesn't reliably return
4984	 * to a non-D0 state anyway.
4985	 */
4986	pci_set_power_state(dev, PCI_D0);
4987
4988	pci_save_state(dev);
4989	/*
4990	 * Disable the device by clearing the Command register, except for
4991	 * INTx-disable which is set.  This not only disables MMIO and I/O port
4992	 * BARs, but also prevents the device from being Bus Master, preventing
4993	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
4994	 * compliant devices, INTx-disable prevents legacy interrupts.
4995	 */
4996	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
4997}
4998
4999static void pci_dev_restore(struct pci_dev *dev)
5000{
5001	const struct pci_error_handlers *err_handler =
5002			dev->driver ? dev->driver->err_handler : NULL;
5003
5004	pci_restore_state(dev);
5005
5006	/*
5007	 * dev->driver->err_handler->reset_done() is protected against
5008	 * races with ->remove() by the device lock, which must be held by
5009	 * the caller.
5010	 */
5011	if (err_handler && err_handler->reset_done)
5012		err_handler->reset_done(dev);
5013}
5014
5015/* dev->reset_methods[] is a 0-terminated list of indices into this array */
5016static const struct pci_reset_fn_method pci_reset_fn_methods[] = {
5017	{ },
5018	{ pci_dev_specific_reset, .name = "device_specific" },
5019	{ pci_dev_acpi_reset, .name = "acpi" },
5020	{ pcie_reset_flr, .name = "flr" },
5021	{ pci_af_flr, .name = "af_flr" },
5022	{ pci_pm_reset, .name = "pm" },
5023	{ pci_reset_bus_function, .name = "bus" },
5024};
5025
5026static ssize_t reset_method_show(struct device *dev,
5027				 struct device_attribute *attr, char *buf)
5028{
5029	struct pci_dev *pdev = to_pci_dev(dev);
5030	ssize_t len = 0;
5031	int i, m;
5032
5033	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5034		m = pdev->reset_methods[i];
5035		if (!m)
5036			break;
5037
5038		len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "",
5039				     pci_reset_fn_methods[m].name);
5040	}
5041
5042	if (len)
5043		len += sysfs_emit_at(buf, len, "\n");
5044
5045	return len;
5046}
5047
5048static int reset_method_lookup(const char *name)
5049{
5050	int m;
5051
5052	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5053		if (sysfs_streq(name, pci_reset_fn_methods[m].name))
5054			return m;
5055	}
5056
5057	return 0;	/* not found */
5058}
5059
5060static ssize_t reset_method_store(struct device *dev,
5061				  struct device_attribute *attr,
5062				  const char *buf, size_t count)
5063{
5064	struct pci_dev *pdev = to_pci_dev(dev);
5065	char *options, *name;
5066	int m, n;
5067	u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 };
5068
5069	if (sysfs_streq(buf, "")) {
5070		pdev->reset_methods[0] = 0;
5071		pci_warn(pdev, "All device reset methods disabled by user");
5072		return count;
5073	}
5074
5075	if (sysfs_streq(buf, "default")) {
5076		pci_init_reset_methods(pdev);
5077		return count;
5078	}
5079
5080	options = kstrndup(buf, count, GFP_KERNEL);
5081	if (!options)
5082		return -ENOMEM;
5083
5084	n = 0;
5085	while ((name = strsep(&options, " ")) != NULL) {
5086		if (sysfs_streq(name, ""))
5087			continue;
5088
5089		name = strim(name);
5090
5091		m = reset_method_lookup(name);
5092		if (!m) {
5093			pci_err(pdev, "Invalid reset method '%s'", name);
5094			goto error;
5095		}
5096
5097		if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) {
5098			pci_err(pdev, "Unsupported reset method '%s'", name);
5099			goto error;
5100		}
5101
5102		if (n == PCI_NUM_RESET_METHODS - 1) {
5103			pci_err(pdev, "Too many reset methods\n");
5104			goto error;
5105		}
5106
5107		reset_methods[n++] = m;
5108	}
5109
5110	reset_methods[n] = 0;
5111
5112	/* Warn if dev-specific supported but not highest priority */
5113	if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 &&
5114	    reset_methods[0] != 1)
5115		pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user");
5116	memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods));
5117	kfree(options);
5118	return count;
5119
5120error:
5121	/* Leave previous methods unchanged */
5122	kfree(options);
5123	return -EINVAL;
5124}
5125static DEVICE_ATTR_RW(reset_method);
5126
5127static struct attribute *pci_dev_reset_method_attrs[] = {
5128	&dev_attr_reset_method.attr,
5129	NULL,
5130};
5131
5132static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj,
5133						    struct attribute *a, int n)
5134{
5135	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
5136
5137	if (!pci_reset_supported(pdev))
5138		return 0;
5139
5140	return a->mode;
5141}
5142
5143const struct attribute_group pci_dev_reset_method_attr_group = {
5144	.attrs = pci_dev_reset_method_attrs,
5145	.is_visible = pci_dev_reset_method_attr_is_visible,
5146};
5147
5148/**
5149 * __pci_reset_function_locked - reset a PCI device function while holding
5150 * the @dev mutex lock.
5151 * @dev: PCI device to reset
5152 *
5153 * Some devices allow an individual function to be reset without affecting
5154 * other functions in the same device.  The PCI device must be responsive
5155 * to PCI config space in order to use this function.
5156 *
5157 * The device function is presumed to be unused and the caller is holding
5158 * the device mutex lock when this function is called.
5159 *
5160 * Resetting the device will make the contents of PCI configuration space
5161 * random, so any caller of this must be prepared to reinitialise the
5162 * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
5163 * etc.
5164 *
5165 * Returns 0 if the device function was successfully reset or negative if the
5166 * device doesn't support resetting a single function.
5167 */
5168int __pci_reset_function_locked(struct pci_dev *dev)
5169{
5170	int i, m, rc;
5171
5172	might_sleep();
5173
5174	/*
5175	 * A reset method returns -ENOTTY if it doesn't support this device and
5176	 * we should try the next method.
5177	 *
5178	 * If it returns 0 (success), we're finished.  If it returns any other
5179	 * error, we're also finished: this indicates that further reset
5180	 * mechanisms might be broken on the device.
5181	 */
5182	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5183		m = dev->reset_methods[i];
5184		if (!m)
5185			return -ENOTTY;
5186
5187		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET);
5188		if (!rc)
5189			return 0;
5190		if (rc != -ENOTTY)
5191			return rc;
5192	}
5193
5194	return -ENOTTY;
 
 
 
 
 
5195}
5196EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
5197
5198/**
5199 * pci_init_reset_methods - check whether device can be safely reset
5200 * and store supported reset mechanisms.
5201 * @dev: PCI device to check for reset mechanisms
5202 *
5203 * Some devices allow an individual function to be reset without affecting
5204 * other functions in the same device.  The PCI device must be in D0-D3hot
5205 * state.
5206 *
5207 * Stores reset mechanisms supported by device in reset_methods byte array
5208 * which is a member of struct pci_dev.
5209 */
5210void pci_init_reset_methods(struct pci_dev *dev)
5211{
5212	int m, i, rc;
5213
5214	BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS);
5215
5216	might_sleep();
5217
5218	i = 0;
5219	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5220		rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE);
5221		if (!rc)
5222			dev->reset_methods[i++] = m;
5223		else if (rc != -ENOTTY)
5224			break;
5225	}
 
 
 
5226
5227	dev->reset_methods[i] = 0;
5228}
5229
5230/**
5231 * pci_reset_function - quiesce and reset a PCI device function
5232 * @dev: PCI device to reset
5233 *
5234 * Some devices allow an individual function to be reset without affecting
5235 * other functions in the same device.  The PCI device must be responsive
5236 * to PCI config space in order to use this function.
5237 *
5238 * This function does not just reset the PCI portion of a device, but
5239 * clears all the state associated with the device.  This function differs
5240 * from __pci_reset_function_locked() in that it saves and restores device state
5241 * over the reset and takes the PCI device lock.
5242 *
5243 * Returns 0 if the device function was successfully reset or negative if the
5244 * device doesn't support resetting a single function.
5245 */
5246int pci_reset_function(struct pci_dev *dev)
5247{
5248	int rc;
5249
5250	if (!pci_reset_supported(dev))
5251		return -ENOTTY;
5252
5253	pci_dev_lock(dev);
5254	pci_dev_save_and_disable(dev);
5255
5256	rc = __pci_reset_function_locked(dev);
5257
5258	pci_dev_restore(dev);
5259	pci_dev_unlock(dev);
5260
5261	return rc;
5262}
5263EXPORT_SYMBOL_GPL(pci_reset_function);
5264
5265/**
5266 * pci_reset_function_locked - quiesce and reset a PCI device function
5267 * @dev: PCI device to reset
5268 *
5269 * Some devices allow an individual function to be reset without affecting
5270 * other functions in the same device.  The PCI device must be responsive
5271 * to PCI config space in order to use this function.
5272 *
5273 * This function does not just reset the PCI portion of a device, but
5274 * clears all the state associated with the device.  This function differs
5275 * from __pci_reset_function_locked() in that it saves and restores device state
5276 * over the reset.  It also differs from pci_reset_function() in that it
5277 * requires the PCI device lock to be held.
5278 *
5279 * Returns 0 if the device function was successfully reset or negative if the
5280 * device doesn't support resetting a single function.
5281 */
5282int pci_reset_function_locked(struct pci_dev *dev)
5283{
5284	int rc;
5285
5286	if (!pci_reset_supported(dev))
5287		return -ENOTTY;
5288
5289	pci_dev_save_and_disable(dev);
5290
5291	rc = __pci_reset_function_locked(dev);
5292
5293	pci_dev_restore(dev);
5294
5295	return rc;
5296}
5297EXPORT_SYMBOL_GPL(pci_reset_function_locked);
5298
5299/**
5300 * pci_try_reset_function - quiesce and reset a PCI device function
5301 * @dev: PCI device to reset
5302 *
5303 * Same as above, except return -EAGAIN if unable to lock device.
5304 */
5305int pci_try_reset_function(struct pci_dev *dev)
5306{
5307	int rc;
5308
5309	if (!pci_reset_supported(dev))
5310		return -ENOTTY;
5311
5312	if (!pci_dev_trylock(dev))
5313		return -EAGAIN;
5314
5315	pci_dev_save_and_disable(dev);
5316	rc = __pci_reset_function_locked(dev);
5317	pci_dev_restore(dev);
5318	pci_dev_unlock(dev);
5319
5320	return rc;
5321}
5322EXPORT_SYMBOL_GPL(pci_try_reset_function);
5323
5324/* Do any devices on or below this bus prevent a bus reset? */
5325static bool pci_bus_resettable(struct pci_bus *bus)
5326{
5327	struct pci_dev *dev;
5328
5329
5330	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5331		return false;
5332
5333	list_for_each_entry(dev, &bus->devices, bus_list) {
5334		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5335		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5336			return false;
5337	}
5338
5339	return true;
5340}
5341
5342/* Lock devices from the top of the tree down */
5343static void pci_bus_lock(struct pci_bus *bus)
5344{
5345	struct pci_dev *dev;
5346
5347	list_for_each_entry(dev, &bus->devices, bus_list) {
5348		pci_dev_lock(dev);
5349		if (dev->subordinate)
5350			pci_bus_lock(dev->subordinate);
5351	}
5352}
5353
5354/* Unlock devices from the bottom of the tree up */
5355static void pci_bus_unlock(struct pci_bus *bus)
5356{
5357	struct pci_dev *dev;
5358
5359	list_for_each_entry(dev, &bus->devices, bus_list) {
5360		if (dev->subordinate)
5361			pci_bus_unlock(dev->subordinate);
5362		pci_dev_unlock(dev);
5363	}
5364}
5365
5366/* Return 1 on successful lock, 0 on contention */
5367static int pci_bus_trylock(struct pci_bus *bus)
5368{
5369	struct pci_dev *dev;
5370
5371	list_for_each_entry(dev, &bus->devices, bus_list) {
5372		if (!pci_dev_trylock(dev))
5373			goto unlock;
5374		if (dev->subordinate) {
5375			if (!pci_bus_trylock(dev->subordinate)) {
5376				pci_dev_unlock(dev);
5377				goto unlock;
5378			}
5379		}
5380	}
5381	return 1;
5382
5383unlock:
5384	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
5385		if (dev->subordinate)
5386			pci_bus_unlock(dev->subordinate);
5387		pci_dev_unlock(dev);
5388	}
5389	return 0;
5390}
5391
5392/* Do any devices on or below this slot prevent a bus reset? */
5393static bool pci_slot_resettable(struct pci_slot *slot)
5394{
5395	struct pci_dev *dev;
5396
5397	if (slot->bus->self &&
5398	    (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5399		return false;
5400
5401	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5402		if (!dev->slot || dev->slot != slot)
5403			continue;
5404		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5405		    (dev->subordinate && !pci_bus_resettable(dev->subordinate)))
5406			return false;
5407	}
5408
5409	return true;
5410}
5411
5412/* Lock devices from the top of the tree down */
5413static void pci_slot_lock(struct pci_slot *slot)
5414{
5415	struct pci_dev *dev;
5416
5417	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5418		if (!dev->slot || dev->slot != slot)
5419			continue;
5420		pci_dev_lock(dev);
5421		if (dev->subordinate)
5422			pci_bus_lock(dev->subordinate);
5423	}
5424}
5425
5426/* Unlock devices from the bottom of the tree up */
5427static void pci_slot_unlock(struct pci_slot *slot)
5428{
5429	struct pci_dev *dev;
5430
5431	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5432		if (!dev->slot || dev->slot != slot)
5433			continue;
5434		if (dev->subordinate)
5435			pci_bus_unlock(dev->subordinate);
5436		pci_dev_unlock(dev);
5437	}
5438}
5439
5440/* Return 1 on successful lock, 0 on contention */
5441static int pci_slot_trylock(struct pci_slot *slot)
5442{
5443	struct pci_dev *dev;
5444
5445	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5446		if (!dev->slot || dev->slot != slot)
5447			continue;
5448		if (!pci_dev_trylock(dev))
5449			goto unlock;
5450		if (dev->subordinate) {
5451			if (!pci_bus_trylock(dev->subordinate)) {
5452				pci_dev_unlock(dev);
5453				goto unlock;
5454			}
5455		}
5456	}
5457	return 1;
5458
5459unlock:
5460	list_for_each_entry_continue_reverse(dev,
5461					     &slot->bus->devices, bus_list) {
5462		if (!dev->slot || dev->slot != slot)
5463			continue;
5464		if (dev->subordinate)
5465			pci_bus_unlock(dev->subordinate);
5466		pci_dev_unlock(dev);
5467	}
5468	return 0;
5469}
5470
5471/*
5472 * Save and disable devices from the top of the tree down while holding
5473 * the @dev mutex lock for the entire tree.
5474 */
5475static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5476{
5477	struct pci_dev *dev;
5478
5479	list_for_each_entry(dev, &bus->devices, bus_list) {
5480		pci_dev_save_and_disable(dev);
5481		if (dev->subordinate)
5482			pci_bus_save_and_disable_locked(dev->subordinate);
5483	}
5484}
5485
5486/*
5487 * Restore devices from top of the tree down while holding @dev mutex lock
5488 * for the entire tree.  Parent bridges need to be restored before we can
5489 * get to subordinate devices.
5490 */
5491static void pci_bus_restore_locked(struct pci_bus *bus)
5492{
5493	struct pci_dev *dev;
5494
5495	list_for_each_entry(dev, &bus->devices, bus_list) {
5496		pci_dev_restore(dev);
5497		if (dev->subordinate)
5498			pci_bus_restore_locked(dev->subordinate);
5499	}
5500}
5501
5502/*
5503 * Save and disable devices from the top of the tree down while holding
5504 * the @dev mutex lock for the entire tree.
5505 */
5506static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5507{
5508	struct pci_dev *dev;
5509
5510	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5511		if (!dev->slot || dev->slot != slot)
5512			continue;
5513		pci_dev_save_and_disable(dev);
5514		if (dev->subordinate)
5515			pci_bus_save_and_disable_locked(dev->subordinate);
5516	}
5517}
5518
5519/*
5520 * Restore devices from top of the tree down while holding @dev mutex lock
5521 * for the entire tree.  Parent bridges need to be restored before we can
5522 * get to subordinate devices.
5523 */
5524static void pci_slot_restore_locked(struct pci_slot *slot)
5525{
5526	struct pci_dev *dev;
5527
5528	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5529		if (!dev->slot || dev->slot != slot)
5530			continue;
5531		pci_dev_restore(dev);
5532		if (dev->subordinate)
5533			pci_bus_restore_locked(dev->subordinate);
5534	}
5535}
5536
5537static int pci_slot_reset(struct pci_slot *slot, bool probe)
5538{
5539	int rc;
5540
5541	if (!slot || !pci_slot_resettable(slot))
5542		return -ENOTTY;
5543
5544	if (!probe)
5545		pci_slot_lock(slot);
5546
5547	might_sleep();
5548
5549	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
5550
5551	if (!probe)
5552		pci_slot_unlock(slot);
5553
5554	return rc;
5555}
5556
5557/**
5558 * pci_probe_reset_slot - probe whether a PCI slot can be reset
5559 * @slot: PCI slot to probe
5560 *
5561 * Return 0 if slot can be reset, negative if a slot reset is not supported.
5562 */
5563int pci_probe_reset_slot(struct pci_slot *slot)
5564{
5565	return pci_slot_reset(slot, PCI_RESET_PROBE);
5566}
5567EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5568
5569/**
5570 * __pci_reset_slot - Try to reset a PCI slot
5571 * @slot: PCI slot to reset
5572 *
5573 * A PCI bus may host multiple slots, each slot may support a reset mechanism
5574 * independent of other slots.  For instance, some slots may support slot power
5575 * control.  In the case of a 1:1 bus to slot architecture, this function may
5576 * wrap the bus reset to avoid spurious slot related events such as hotplug.
5577 * Generally a slot reset should be attempted before a bus reset.  All of the
5578 * function of the slot and any subordinate buses behind the slot are reset
5579 * through this function.  PCI config space of all devices in the slot and
5580 * behind the slot is saved before and restored after reset.
5581 *
5582 * Same as above except return -EAGAIN if the slot cannot be locked
5583 */
5584static int __pci_reset_slot(struct pci_slot *slot)
5585{
5586	int rc;
5587
5588	rc = pci_slot_reset(slot, PCI_RESET_PROBE);
5589	if (rc)
5590		return rc;
5591
5592	if (pci_slot_trylock(slot)) {
5593		pci_slot_save_and_disable_locked(slot);
5594		might_sleep();
5595		rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET);
5596		pci_slot_restore_locked(slot);
5597		pci_slot_unlock(slot);
5598	} else
5599		rc = -EAGAIN;
5600
5601	return rc;
5602}
5603
5604static int pci_bus_reset(struct pci_bus *bus, bool probe)
5605{
5606	int ret;
5607
5608	if (!bus->self || !pci_bus_resettable(bus))
5609		return -ENOTTY;
5610
5611	if (probe)
5612		return 0;
5613
5614	pci_bus_lock(bus);
5615
5616	might_sleep();
5617
5618	ret = pci_bridge_secondary_bus_reset(bus->self);
5619
5620	pci_bus_unlock(bus);
5621
5622	return ret;
5623}
5624
5625/**
5626 * pci_bus_error_reset - reset the bridge's subordinate bus
5627 * @bridge: The parent device that connects to the bus to reset
5628 *
5629 * This function will first try to reset the slots on this bus if the method is
5630 * available. If slot reset fails or is not available, this will fall back to a
5631 * secondary bus reset.
5632 */
5633int pci_bus_error_reset(struct pci_dev *bridge)
5634{
5635	struct pci_bus *bus = bridge->subordinate;
5636	struct pci_slot *slot;
5637
5638	if (!bus)
5639		return -ENOTTY;
5640
5641	mutex_lock(&pci_slot_mutex);
5642	if (list_empty(&bus->slots))
5643		goto bus_reset;
5644
5645	list_for_each_entry(slot, &bus->slots, list)
5646		if (pci_probe_reset_slot(slot))
5647			goto bus_reset;
5648
5649	list_for_each_entry(slot, &bus->slots, list)
5650		if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
5651			goto bus_reset;
5652
5653	mutex_unlock(&pci_slot_mutex);
5654	return 0;
5655bus_reset:
5656	mutex_unlock(&pci_slot_mutex);
5657	return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET);
5658}
5659
5660/**
5661 * pci_probe_reset_bus - probe whether a PCI bus can be reset
5662 * @bus: PCI bus to probe
5663 *
5664 * Return 0 if bus can be reset, negative if a bus reset is not supported.
5665 */
5666int pci_probe_reset_bus(struct pci_bus *bus)
5667{
5668	return pci_bus_reset(bus, PCI_RESET_PROBE);
5669}
5670EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5671
5672/**
5673 * __pci_reset_bus - Try to reset a PCI bus
5674 * @bus: top level PCI bus to reset
5675 *
5676 * Same as above except return -EAGAIN if the bus cannot be locked
5677 */
5678static int __pci_reset_bus(struct pci_bus *bus)
5679{
5680	int rc;
5681
5682	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
5683	if (rc)
5684		return rc;
5685
5686	if (pci_bus_trylock(bus)) {
5687		pci_bus_save_and_disable_locked(bus);
5688		might_sleep();
5689		rc = pci_bridge_secondary_bus_reset(bus->self);
5690		pci_bus_restore_locked(bus);
5691		pci_bus_unlock(bus);
5692	} else
5693		rc = -EAGAIN;
5694
5695	return rc;
5696}
5697
5698/**
5699 * pci_reset_bus - Try to reset a PCI bus
5700 * @pdev: top level PCI device to reset via slot/bus
5701 *
5702 * Same as above except return -EAGAIN if the bus cannot be locked
5703 */
5704int pci_reset_bus(struct pci_dev *pdev)
5705{
5706	return (!pci_probe_reset_slot(pdev->slot)) ?
5707	    __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
5708}
5709EXPORT_SYMBOL_GPL(pci_reset_bus);
5710
5711/**
5712 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
5713 * @dev: PCI device to query
5714 *
5715 * Returns mmrbc: maximum designed memory read count in bytes or
5716 * appropriate error value.
5717 */
5718int pcix_get_max_mmrbc(struct pci_dev *dev)
5719{
5720	int cap;
5721	u32 stat;
5722
5723	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5724	if (!cap)
5725		return -EINVAL;
5726
5727	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5728		return -EINVAL;
5729
5730	return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat);
5731}
5732EXPORT_SYMBOL(pcix_get_max_mmrbc);
5733
5734/**
5735 * pcix_get_mmrbc - get PCI-X maximum memory read byte count
5736 * @dev: PCI device to query
5737 *
5738 * Returns mmrbc: maximum memory read count in bytes or appropriate error
5739 * value.
5740 */
5741int pcix_get_mmrbc(struct pci_dev *dev)
5742{
5743	int cap;
5744	u16 cmd;
5745
5746	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5747	if (!cap)
5748		return -EINVAL;
5749
5750	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5751		return -EINVAL;
5752
5753	return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5754}
5755EXPORT_SYMBOL(pcix_get_mmrbc);
5756
5757/**
5758 * pcix_set_mmrbc - set PCI-X maximum memory read byte count
5759 * @dev: PCI device to query
5760 * @mmrbc: maximum memory read count in bytes
5761 *    valid values are 512, 1024, 2048, 4096
5762 *
5763 * If possible sets maximum memory read byte count, some bridges have errata
5764 * that prevent this.
5765 */
5766int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
5767{
5768	int cap;
5769	u32 stat, v, o;
5770	u16 cmd;
5771
5772	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
5773		return -EINVAL;
5774
5775	v = ffs(mmrbc) - 10;
5776
5777	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5778	if (!cap)
5779		return -EINVAL;
5780
5781	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5782		return -EINVAL;
5783
5784	if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat))
5785		return -E2BIG;
5786
5787	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5788		return -EINVAL;
5789
5790	o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5791	if (o != v) {
5792		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
5793			return -EIO;
5794
5795		cmd &= ~PCI_X_CMD_MAX_READ;
5796		cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
5797		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
5798			return -EIO;
5799	}
5800	return 0;
5801}
5802EXPORT_SYMBOL(pcix_set_mmrbc);
5803
5804/**
5805 * pcie_get_readrq - get PCI Express read request size
5806 * @dev: PCI device to query
5807 *
5808 * Returns maximum memory read request in bytes or appropriate error value.
5809 */
5810int pcie_get_readrq(struct pci_dev *dev)
5811{
5812	u16 ctl;
5813
5814	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5815
5816	return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl);
5817}
5818EXPORT_SYMBOL(pcie_get_readrq);
5819
5820/**
5821 * pcie_set_readrq - set PCI Express maximum memory read request
5822 * @dev: PCI device to query
5823 * @rq: maximum memory read count in bytes
5824 *    valid values are 128, 256, 512, 1024, 2048, 4096
5825 *
5826 * If possible sets maximum memory read request in bytes
5827 */
5828int pcie_set_readrq(struct pci_dev *dev, int rq)
5829{
5830	u16 v;
5831	int ret;
5832	struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
5833
5834	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
5835		return -EINVAL;
5836
5837	/*
5838	 * If using the "performance" PCIe config, we clamp the read rq
5839	 * size to the max packet size to keep the host bridge from
5840	 * generating requests larger than we can cope with.
5841	 */
5842	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
5843		int mps = pcie_get_mps(dev);
5844
5845		if (mps < rq)
5846			rq = mps;
5847	}
5848
5849	v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, ffs(rq) - 8);
5850
5851	if (bridge->no_inc_mrrs) {
5852		int max_mrrs = pcie_get_readrq(dev);
5853
5854		if (rq > max_mrrs) {
5855			pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs);
5856			return -EINVAL;
5857		}
5858	}
5859
5860	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5861						  PCI_EXP_DEVCTL_READRQ, v);
5862
5863	return pcibios_err_to_errno(ret);
5864}
5865EXPORT_SYMBOL(pcie_set_readrq);
5866
5867/**
5868 * pcie_get_mps - get PCI Express maximum payload size
5869 * @dev: PCI device to query
5870 *
5871 * Returns maximum payload size in bytes
5872 */
5873int pcie_get_mps(struct pci_dev *dev)
5874{
5875	u16 ctl;
5876
5877	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5878
5879	return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl);
5880}
5881EXPORT_SYMBOL(pcie_get_mps);
5882
5883/**
5884 * pcie_set_mps - set PCI Express maximum payload size
5885 * @dev: PCI device to query
5886 * @mps: maximum payload size in bytes
5887 *    valid values are 128, 256, 512, 1024, 2048, 4096
5888 *
5889 * If possible sets maximum payload size
5890 */
5891int pcie_set_mps(struct pci_dev *dev, int mps)
5892{
5893	u16 v;
5894	int ret;
5895
5896	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
5897		return -EINVAL;
5898
5899	v = ffs(mps) - 8;
5900	if (v > dev->pcie_mpss)
5901		return -EINVAL;
5902	v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v);
5903
5904	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5905						  PCI_EXP_DEVCTL_PAYLOAD, v);
5906
5907	return pcibios_err_to_errno(ret);
5908}
5909EXPORT_SYMBOL(pcie_set_mps);
5910
5911static enum pci_bus_speed to_pcie_link_speed(u16 lnksta)
5912{
5913	return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)];
5914}
5915
5916int pcie_link_speed_mbps(struct pci_dev *pdev)
5917{
5918	u16 lnksta;
5919	int err;
5920
5921	err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
5922	if (err)
5923		return err;
5924
5925	switch (to_pcie_link_speed(lnksta)) {
5926	case PCIE_SPEED_2_5GT:
5927		return 2500;
5928	case PCIE_SPEED_5_0GT:
5929		return 5000;
5930	case PCIE_SPEED_8_0GT:
5931		return 8000;
5932	case PCIE_SPEED_16_0GT:
5933		return 16000;
5934	case PCIE_SPEED_32_0GT:
5935		return 32000;
5936	case PCIE_SPEED_64_0GT:
5937		return 64000;
5938	default:
5939		break;
5940	}
5941
5942	return -EINVAL;
5943}
5944EXPORT_SYMBOL(pcie_link_speed_mbps);
5945
5946/**
5947 * pcie_bandwidth_available - determine minimum link settings of a PCIe
5948 *			      device and its bandwidth limitation
5949 * @dev: PCI device to query
5950 * @limiting_dev: storage for device causing the bandwidth limitation
5951 * @speed: storage for speed of limiting device
5952 * @width: storage for width of limiting device
5953 *
5954 * Walk up the PCI device chain and find the point where the minimum
5955 * bandwidth is available.  Return the bandwidth available there and (if
5956 * limiting_dev, speed, and width pointers are supplied) information about
5957 * that point.  The bandwidth returned is in Mb/s, i.e., megabits/second of
5958 * raw bandwidth.
5959 */
5960u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
5961			     enum pci_bus_speed *speed,
5962			     enum pcie_link_width *width)
5963{
5964	u16 lnksta;
5965	enum pci_bus_speed next_speed;
5966	enum pcie_link_width next_width;
5967	u32 bw, next_bw;
5968
5969	if (speed)
5970		*speed = PCI_SPEED_UNKNOWN;
5971	if (width)
5972		*width = PCIE_LNK_WIDTH_UNKNOWN;
5973
5974	bw = 0;
5975
5976	while (dev) {
5977		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
5978
5979		next_speed = to_pcie_link_speed(lnksta);
5980		next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
 
5981
5982		next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
5983
5984		/* Check if current device limits the total bandwidth */
5985		if (!bw || next_bw <= bw) {
5986			bw = next_bw;
5987
5988			if (limiting_dev)
5989				*limiting_dev = dev;
5990			if (speed)
5991				*speed = next_speed;
5992			if (width)
5993				*width = next_width;
5994		}
5995
5996		dev = pci_upstream_bridge(dev);
5997	}
5998
5999	return bw;
6000}
6001EXPORT_SYMBOL(pcie_bandwidth_available);
6002
6003/**
6004 * pcie_get_speed_cap - query for the PCI device's link speed capability
6005 * @dev: PCI device to query
6006 *
6007 * Query the PCI device speed capability.  Return the maximum link speed
6008 * supported by the device.
6009 */
6010enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
6011{
6012	u32 lnkcap2, lnkcap;
6013
6014	/*
6015	 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18.  The
6016	 * implementation note there recommends using the Supported Link
6017	 * Speeds Vector in Link Capabilities 2 when supported.
6018	 *
6019	 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
6020	 * should use the Supported Link Speeds field in Link Capabilities,
6021	 * where only 2.5 GT/s and 5.0 GT/s speeds were defined.
6022	 */
6023	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2);
6024
6025	/* PCIe r3.0-compliant */
6026	if (lnkcap2)
6027		return PCIE_LNKCAP2_SLS2SPEED(lnkcap2);
6028
6029	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6030	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
6031		return PCIE_SPEED_5_0GT;
6032	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
6033		return PCIE_SPEED_2_5GT;
6034
6035	return PCI_SPEED_UNKNOWN;
6036}
6037EXPORT_SYMBOL(pcie_get_speed_cap);
6038
6039/**
6040 * pcie_get_width_cap - query for the PCI device's link width capability
6041 * @dev: PCI device to query
6042 *
6043 * Query the PCI device width capability.  Return the maximum link width
6044 * supported by the device.
6045 */
6046enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
6047{
6048	u32 lnkcap;
6049
6050	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
6051	if (lnkcap)
6052		return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap);
6053
6054	return PCIE_LNK_WIDTH_UNKNOWN;
6055}
6056EXPORT_SYMBOL(pcie_get_width_cap);
6057
6058/**
6059 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
6060 * @dev: PCI device
6061 * @speed: storage for link speed
6062 * @width: storage for link width
6063 *
6064 * Calculate a PCI device's link bandwidth by querying for its link speed
6065 * and width, multiplying them, and applying encoding overhead.  The result
6066 * is in Mb/s, i.e., megabits/second of raw bandwidth.
6067 */
6068u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
6069			   enum pcie_link_width *width)
6070{
6071	*speed = pcie_get_speed_cap(dev);
6072	*width = pcie_get_width_cap(dev);
6073
6074	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
6075		return 0;
6076
6077	return *width * PCIE_SPEED2MBS_ENC(*speed);
6078}
6079
6080/**
6081 * __pcie_print_link_status - Report the PCI device's link speed and width
6082 * @dev: PCI device to query
6083 * @verbose: Print info even when enough bandwidth is available
6084 *
6085 * If the available bandwidth at the device is less than the device is
6086 * capable of, report the device's maximum possible bandwidth and the
6087 * upstream link that limits its performance.  If @verbose, always print
6088 * the available bandwidth, even if the device isn't constrained.
6089 */
6090void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
6091{
6092	enum pcie_link_width width, width_cap;
6093	enum pci_bus_speed speed, speed_cap;
6094	struct pci_dev *limiting_dev = NULL;
6095	u32 bw_avail, bw_cap;
6096
6097	bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap);
6098	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
6099
6100	if (bw_avail >= bw_cap && verbose)
6101		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
6102			 bw_cap / 1000, bw_cap % 1000,
6103			 pci_speed_string(speed_cap), width_cap);
6104	else if (bw_avail < bw_cap)
6105		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
6106			 bw_avail / 1000, bw_avail % 1000,
6107			 pci_speed_string(speed), width,
6108			 limiting_dev ? pci_name(limiting_dev) : "<unknown>",
6109			 bw_cap / 1000, bw_cap % 1000,
6110			 pci_speed_string(speed_cap), width_cap);
6111}
6112
6113/**
6114 * pcie_print_link_status - Report the PCI device's link speed and width
6115 * @dev: PCI device to query
6116 *
6117 * Report the available bandwidth at the device.
6118 */
6119void pcie_print_link_status(struct pci_dev *dev)
6120{
6121	__pcie_print_link_status(dev, true);
6122}
6123EXPORT_SYMBOL(pcie_print_link_status);
6124
6125/**
6126 * pci_select_bars - Make BAR mask from the type of resource
6127 * @dev: the PCI device for which BAR mask is made
6128 * @flags: resource type mask to be selected
6129 *
6130 * This helper routine makes bar mask from the type of resource.
6131 */
6132int pci_select_bars(struct pci_dev *dev, unsigned long flags)
6133{
6134	int i, bars = 0;
6135	for (i = 0; i < PCI_NUM_RESOURCES; i++)
6136		if (pci_resource_flags(dev, i) & flags)
6137			bars |= (1 << i);
6138	return bars;
6139}
6140EXPORT_SYMBOL(pci_select_bars);
6141
6142/* Some architectures require additional programming to enable VGA */
6143static arch_set_vga_state_t arch_set_vga_state;
6144
6145void __init pci_register_set_vga_state(arch_set_vga_state_t func)
6146{
6147	arch_set_vga_state = func;	/* NULL disables */
6148}
6149
6150static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
6151				  unsigned int command_bits, u32 flags)
6152{
6153	if (arch_set_vga_state)
6154		return arch_set_vga_state(dev, decode, command_bits,
6155						flags);
6156	return 0;
6157}
6158
6159/**
6160 * pci_set_vga_state - set VGA decode state on device and parents if requested
6161 * @dev: the PCI device
6162 * @decode: true = enable decoding, false = disable decoding
6163 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
6164 * @flags: traverse ancestors and change bridges
6165 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
6166 */
6167int pci_set_vga_state(struct pci_dev *dev, bool decode,
6168		      unsigned int command_bits, u32 flags)
6169{
6170	struct pci_bus *bus;
6171	struct pci_dev *bridge;
6172	u16 cmd;
6173	int rc;
6174
6175	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
6176
6177	/* ARCH specific VGA enables */
6178	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
6179	if (rc)
6180		return rc;
6181
6182	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
6183		pci_read_config_word(dev, PCI_COMMAND, &cmd);
6184		if (decode)
6185			cmd |= command_bits;
6186		else
6187			cmd &= ~command_bits;
6188		pci_write_config_word(dev, PCI_COMMAND, cmd);
6189	}
6190
6191	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
6192		return 0;
6193
6194	bus = dev->bus;
6195	while (bus) {
6196		bridge = bus->self;
6197		if (bridge) {
6198			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
6199					     &cmd);
6200			if (decode)
6201				cmd |= PCI_BRIDGE_CTL_VGA;
6202			else
6203				cmd &= ~PCI_BRIDGE_CTL_VGA;
6204			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
6205					      cmd);
6206		}
6207		bus = bus->parent;
6208	}
6209	return 0;
6210}
6211
6212#ifdef CONFIG_ACPI
6213bool pci_pr3_present(struct pci_dev *pdev)
6214{
6215	struct acpi_device *adev;
6216
6217	if (acpi_disabled)
6218		return false;
6219
6220	adev = ACPI_COMPANION(&pdev->dev);
6221	if (!adev)
6222		return false;
6223
6224	return adev->power.flags.power_resources &&
6225		acpi_has_method(adev->handle, "_PR3");
6226}
6227EXPORT_SYMBOL_GPL(pci_pr3_present);
6228#endif
6229
6230/**
6231 * pci_add_dma_alias - Add a DMA devfn alias for a device
6232 * @dev: the PCI device for which alias is added
6233 * @devfn_from: alias slot and function
6234 * @nr_devfns: number of subsequent devfns to alias
6235 *
6236 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
6237 * which is used to program permissible bus-devfn source addresses for DMA
6238 * requests in an IOMMU.  These aliases factor into IOMMU group creation
6239 * and are useful for devices generating DMA requests beyond or different
6240 * from their logical bus-devfn.  Examples include device quirks where the
6241 * device simply uses the wrong devfn, as well as non-transparent bridges
6242 * where the alias may be a proxy for devices in another domain.
6243 *
6244 * IOMMU group creation is performed during device discovery or addition,
6245 * prior to any potential DMA mapping and therefore prior to driver probing
6246 * (especially for userspace assigned devices where IOMMU group definition
6247 * cannot be left as a userspace activity).  DMA aliases should therefore
6248 * be configured via quirks, such as the PCI fixup header quirk.
6249 */
6250void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from,
6251		       unsigned int nr_devfns)
6252{
6253	int devfn_to;
6254
6255	nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from);
6256	devfn_to = devfn_from + nr_devfns - 1;
6257
6258	if (!dev->dma_alias_mask)
6259		dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
6260	if (!dev->dma_alias_mask) {
6261		pci_warn(dev, "Unable to allocate DMA alias mask\n");
6262		return;
6263	}
6264
6265	bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns);
6266
6267	if (nr_devfns == 1)
6268		pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
6269				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
6270	else if (nr_devfns > 1)
6271		pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
6272				PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
6273				PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
6274}
6275
6276bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
6277{
6278	return (dev1->dma_alias_mask &&
6279		test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
6280	       (dev2->dma_alias_mask &&
6281		test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
6282	       pci_real_dma_dev(dev1) == dev2 ||
6283	       pci_real_dma_dev(dev2) == dev1;
6284}
6285
6286bool pci_device_is_present(struct pci_dev *pdev)
6287{
6288	u32 v;
6289
6290	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
6291	pdev = pci_physfn(pdev);
6292	if (pci_dev_is_disconnected(pdev))
6293		return false;
6294	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
6295}
6296EXPORT_SYMBOL_GPL(pci_device_is_present);
6297
6298void pci_ignore_hotplug(struct pci_dev *dev)
6299{
6300	struct pci_dev *bridge = dev->bus->self;
6301
6302	dev->ignore_hotplug = 1;
6303	/* Propagate the "ignore hotplug" setting to the parent bridge. */
6304	if (bridge)
6305		bridge->ignore_hotplug = 1;
6306}
6307EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
6308
6309/**
6310 * pci_real_dma_dev - Get PCI DMA device for PCI device
6311 * @dev: the PCI device that may have a PCI DMA alias
6312 *
6313 * Permits the platform to provide architecture-specific functionality to
6314 * devices needing to alias DMA to another PCI device on another PCI bus. If
6315 * the PCI device is on the same bus, it is recommended to use
6316 * pci_add_dma_alias(). This is the default implementation. Architecture
6317 * implementations can override this.
6318 */
6319struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
6320{
6321	return dev;
6322}
6323
6324resource_size_t __weak pcibios_default_alignment(void)
6325{
6326	return 0;
6327}
6328
6329/*
6330 * Arches that don't want to expose struct resource to userland as-is in
6331 * sysfs and /proc can implement their own pci_resource_to_user().
6332 */
6333void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
6334				 const struct resource *rsrc,
6335				 resource_size_t *start, resource_size_t *end)
6336{
6337	*start = rsrc->start;
6338	*end = rsrc->end;
6339}
6340
6341static char *resource_alignment_param;
6342static DEFINE_SPINLOCK(resource_alignment_lock);
6343
6344/**
6345 * pci_specified_resource_alignment - get resource alignment specified by user.
6346 * @dev: the PCI device to get
6347 * @resize: whether or not to change resources' size when reassigning alignment
6348 *
6349 * RETURNS: Resource alignment if it is specified.
6350 *          Zero if it is not specified.
6351 */
6352static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
6353							bool *resize)
6354{
6355	int align_order, count;
6356	resource_size_t align = pcibios_default_alignment();
6357	const char *p;
6358	int ret;
6359
6360	spin_lock(&resource_alignment_lock);
6361	p = resource_alignment_param;
6362	if (!p || !*p)
6363		goto out;
6364	if (pci_has_flag(PCI_PROBE_ONLY)) {
6365		align = 0;
6366		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
6367		goto out;
6368	}
6369
6370	while (*p) {
6371		count = 0;
6372		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
6373		    p[count] == '@') {
6374			p += count + 1;
6375			if (align_order > 63) {
6376				pr_err("PCI: Invalid requested alignment (order %d)\n",
6377				       align_order);
6378				align_order = PAGE_SHIFT;
6379			}
6380		} else {
6381			align_order = PAGE_SHIFT;
6382		}
6383
6384		ret = pci_dev_str_match(dev, p, &p);
6385		if (ret == 1) {
6386			*resize = true;
6387			align = 1ULL << align_order;
6388			break;
6389		} else if (ret < 0) {
6390			pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
6391			       p);
6392			break;
6393		}
6394
6395		if (*p != ';' && *p != ',') {
6396			/* End of param or invalid format */
6397			break;
6398		}
6399		p++;
6400	}
6401out:
6402	spin_unlock(&resource_alignment_lock);
6403	return align;
6404}
6405
6406static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
6407					   resource_size_t align, bool resize)
6408{
6409	struct resource *r = &dev->resource[bar];
6410	const char *r_name = pci_resource_name(dev, bar);
6411	resource_size_t size;
6412
6413	if (!(r->flags & IORESOURCE_MEM))
6414		return;
6415
6416	if (r->flags & IORESOURCE_PCI_FIXED) {
6417		pci_info(dev, "%s %pR: ignoring requested alignment %#llx\n",
6418			 r_name, r, (unsigned long long)align);
6419		return;
6420	}
6421
6422	size = resource_size(r);
6423	if (size >= align)
6424		return;
6425
6426	/*
6427	 * Increase the alignment of the resource.  There are two ways we
6428	 * can do this:
6429	 *
6430	 * 1) Increase the size of the resource.  BARs are aligned on their
6431	 *    size, so when we reallocate space for this resource, we'll
6432	 *    allocate it with the larger alignment.  This also prevents
6433	 *    assignment of any other BARs inside the alignment region, so
6434	 *    if we're requesting page alignment, this means no other BARs
6435	 *    will share the page.
6436	 *
6437	 *    The disadvantage is that this makes the resource larger than
6438	 *    the hardware BAR, which may break drivers that compute things
6439	 *    based on the resource size, e.g., to find registers at a
6440	 *    fixed offset before the end of the BAR.
6441	 *
6442	 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
6443	 *    set r->start to the desired alignment.  By itself this
6444	 *    doesn't prevent other BARs being put inside the alignment
6445	 *    region, but if we realign *every* resource of every device in
6446	 *    the system, none of them will share an alignment region.
6447	 *
6448	 * When the user has requested alignment for only some devices via
6449	 * the "pci=resource_alignment" argument, "resize" is true and we
6450	 * use the first method.  Otherwise we assume we're aligning all
6451	 * devices and we use the second.
6452	 */
6453
6454	pci_info(dev, "%s %pR: requesting alignment to %#llx\n",
6455		 r_name, r, (unsigned long long)align);
6456
6457	if (resize) {
6458		r->start = 0;
6459		r->end = align - 1;
6460	} else {
6461		r->flags &= ~IORESOURCE_SIZEALIGN;
6462		r->flags |= IORESOURCE_STARTALIGN;
6463		r->start = align;
6464		r->end = r->start + size - 1;
6465	}
6466	r->flags |= IORESOURCE_UNSET;
6467}
6468
6469/*
6470 * This function disables memory decoding and releases memory resources
6471 * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
6472 * It also rounds up size to specified alignment.
6473 * Later on, the kernel will assign page-aligned memory resource back
6474 * to the device.
6475 */
6476void pci_reassigndev_resource_alignment(struct pci_dev *dev)
6477{
6478	int i;
6479	struct resource *r;
6480	resource_size_t align;
6481	u16 command;
6482	bool resize = false;
6483
6484	/*
6485	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
6486	 * 3.4.1.11.  Their resources are allocated from the space
6487	 * described by the VF BARx register in the PF's SR-IOV capability.
6488	 * We can't influence their alignment here.
6489	 */
6490	if (dev->is_virtfn)
6491		return;
6492
6493	/* check if specified PCI is target device to reassign */
6494	align = pci_specified_resource_alignment(dev, &resize);
6495	if (!align)
6496		return;
6497
6498	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6499	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6500		pci_warn(dev, "Can't reassign resources to host bridge\n");
6501		return;
6502	}
6503
6504	pci_read_config_word(dev, PCI_COMMAND, &command);
6505	command &= ~PCI_COMMAND_MEMORY;
6506	pci_write_config_word(dev, PCI_COMMAND, command);
6507
6508	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6509		pci_request_resource_alignment(dev, i, align, resize);
6510
6511	/*
6512	 * Need to disable bridge's resource window,
6513	 * to enable the kernel to reassign new resource
6514	 * window later on.
6515	 */
6516	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
6517		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6518			r = &dev->resource[i];
6519			if (!(r->flags & IORESOURCE_MEM))
6520				continue;
6521			r->flags |= IORESOURCE_UNSET;
6522			r->end = resource_size(r) - 1;
6523			r->start = 0;
6524		}
6525		pci_disable_bridge_window(dev);
6526	}
6527}
6528
6529static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
6530{
6531	size_t count = 0;
6532
6533	spin_lock(&resource_alignment_lock);
6534	if (resource_alignment_param)
6535		count = sysfs_emit(buf, "%s\n", resource_alignment_param);
6536	spin_unlock(&resource_alignment_lock);
6537
6538	return count;
6539}
6540
6541static ssize_t resource_alignment_store(const struct bus_type *bus,
6542					const char *buf, size_t count)
6543{
6544	char *param, *old, *end;
6545
6546	if (count >= (PAGE_SIZE - 1))
6547		return -EINVAL;
6548
6549	param = kstrndup(buf, count, GFP_KERNEL);
6550	if (!param)
6551		return -ENOMEM;
6552
6553	end = strchr(param, '\n');
6554	if (end)
6555		*end = '\0';
6556
6557	spin_lock(&resource_alignment_lock);
6558	old = resource_alignment_param;
6559	if (strlen(param)) {
6560		resource_alignment_param = param;
6561	} else {
6562		kfree(param);
6563		resource_alignment_param = NULL;
6564	}
6565	spin_unlock(&resource_alignment_lock);
6566
6567	kfree(old);
6568
6569	return count;
6570}
6571
6572static BUS_ATTR_RW(resource_alignment);
6573
6574static int __init pci_resource_alignment_sysfs_init(void)
6575{
6576	return bus_create_file(&pci_bus_type,
6577					&bus_attr_resource_alignment);
6578}
6579late_initcall(pci_resource_alignment_sysfs_init);
6580
6581static void pci_no_domains(void)
6582{
6583#ifdef CONFIG_PCI_DOMAINS
6584	pci_domains_supported = 0;
6585#endif
6586}
6587
6588#ifdef CONFIG_PCI_DOMAINS_GENERIC
6589static DEFINE_IDA(pci_domain_nr_static_ida);
6590static DEFINE_IDA(pci_domain_nr_dynamic_ida);
6591
6592static void of_pci_reserve_static_domain_nr(void)
6593{
6594	struct device_node *np;
6595	int domain_nr;
6596
6597	for_each_node_by_type(np, "pci") {
6598		domain_nr = of_get_pci_domain_nr(np);
6599		if (domain_nr < 0)
6600			continue;
6601		/*
6602		 * Permanently allocate domain_nr in dynamic_ida
6603		 * to prevent it from dynamic allocation.
6604		 */
6605		ida_alloc_range(&pci_domain_nr_dynamic_ida,
6606				domain_nr, domain_nr, GFP_KERNEL);
6607	}
6608}
6609
6610static int of_pci_bus_find_domain_nr(struct device *parent)
6611{
6612	static bool static_domains_reserved = false;
6613	int domain_nr;
6614
6615	/* On the first call scan device tree for static allocations. */
6616	if (!static_domains_reserved) {
6617		of_pci_reserve_static_domain_nr();
6618		static_domains_reserved = true;
6619	}
6620
6621	if (parent) {
6622		/*
6623		 * If domain is in DT, allocate it in static IDA.  This
6624		 * prevents duplicate static allocations in case of errors
6625		 * in DT.
6626		 */
6627		domain_nr = of_get_pci_domain_nr(parent->of_node);
6628		if (domain_nr >= 0)
6629			return ida_alloc_range(&pci_domain_nr_static_ida,
6630					       domain_nr, domain_nr,
6631					       GFP_KERNEL);
6632	}
6633
6634	/*
6635	 * If domain was not specified in DT, choose a free ID from dynamic
6636	 * allocations. All domain numbers from DT are permanently in
6637	 * dynamic allocations to prevent assigning them to other DT nodes
6638	 * without static domain.
6639	 */
6640	return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL);
6641}
6642
6643static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6644{
6645	if (bus->domain_nr < 0)
6646		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6647
6648	/* Release domain from IDA where it was allocated. */
6649	if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr)
6650		ida_free(&pci_domain_nr_static_ida, bus->domain_nr);
6651	else
6652		ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr);
6653}
6654
6655int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6656{
6657	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6658			       acpi_pci_bus_find_domain_nr(bus);
6659}
6660
6661void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6662{
6663	if (!acpi_disabled)
6664		return;
6665	of_pci_bus_release_domain_nr(bus, parent);
6666}
6667#endif
6668
6669/**
6670 * pci_ext_cfg_avail - can we access extended PCI config space?
6671 *
6672 * Returns 1 if we can access PCI extended config space (offsets
6673 * greater than 0xff). This is the default implementation. Architecture
6674 * implementations can override this.
6675 */
6676int __weak pci_ext_cfg_avail(void)
6677{
6678	return 1;
6679}
6680
6681void __weak pci_fixup_cardbus(struct pci_bus *bus)
6682{
6683}
6684EXPORT_SYMBOL(pci_fixup_cardbus);
6685
6686static int __init pci_setup(char *str)
6687{
6688	while (str) {
6689		char *k = strchr(str, ',');
6690		if (k)
6691			*k++ = 0;
6692		if (*str && (str = pcibios_setup(str)) && *str) {
6693			if (!strcmp(str, "nomsi")) {
6694				pci_no_msi();
6695			} else if (!strncmp(str, "noats", 5)) {
6696				pr_info("PCIe: ATS is disabled\n");
6697				pcie_ats_disabled = true;
6698			} else if (!strcmp(str, "noaer")) {
6699				pci_no_aer();
6700			} else if (!strcmp(str, "earlydump")) {
6701				pci_early_dump = true;
6702			} else if (!strncmp(str, "realloc=", 8)) {
6703				pci_realloc_get_opt(str + 8);
6704			} else if (!strncmp(str, "realloc", 7)) {
6705				pci_realloc_get_opt("on");
6706			} else if (!strcmp(str, "nodomains")) {
6707				pci_no_domains();
6708			} else if (!strncmp(str, "noari", 5)) {
6709				pcie_ari_disabled = true;
6710			} else if (!strncmp(str, "cbiosize=", 9)) {
6711				pci_cardbus_io_size = memparse(str + 9, &str);
6712			} else if (!strncmp(str, "cbmemsize=", 10)) {
6713				pci_cardbus_mem_size = memparse(str + 10, &str);
6714			} else if (!strncmp(str, "resource_alignment=", 19)) {
6715				resource_alignment_param = str + 19;
6716			} else if (!strncmp(str, "ecrc=", 5)) {
6717				pcie_ecrc_get_policy(str + 5);
6718			} else if (!strncmp(str, "hpiosize=", 9)) {
6719				pci_hotplug_io_size = memparse(str + 9, &str);
6720			} else if (!strncmp(str, "hpmmiosize=", 11)) {
6721				pci_hotplug_mmio_size = memparse(str + 11, &str);
6722			} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
6723				pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
6724			} else if (!strncmp(str, "hpmemsize=", 10)) {
6725				pci_hotplug_mmio_size = memparse(str + 10, &str);
6726				pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
6727			} else if (!strncmp(str, "hpbussize=", 10)) {
6728				pci_hotplug_bus_size =
6729					simple_strtoul(str + 10, &str, 0);
6730				if (pci_hotplug_bus_size > 0xff)
6731					pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
6732			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
6733				pcie_bus_config = PCIE_BUS_TUNE_OFF;
6734			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
6735				pcie_bus_config = PCIE_BUS_SAFE;
6736			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
6737				pcie_bus_config = PCIE_BUS_PERFORMANCE;
6738			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
6739				pcie_bus_config = PCIE_BUS_PEER2PEER;
6740			} else if (!strncmp(str, "pcie_scan_all", 13)) {
6741				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
6742			} else if (!strncmp(str, "disable_acs_redir=", 18)) {
6743				disable_acs_redir_param = str + 18;
6744			} else {
6745				pr_err("PCI: Unknown option `%s'\n", str);
6746			}
6747		}
6748		str = k;
6749	}
6750	return 0;
6751}
6752early_param("pci", pci_setup);
6753
6754/*
6755 * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
6756 * in pci_setup(), above, to point to data in the __initdata section which
6757 * will be freed after the init sequence is complete. We can't allocate memory
6758 * in pci_setup() because some architectures do not have any memory allocation
6759 * service available during an early_param() call. So we allocate memory and
6760 * copy the variable here before the init section is freed.
6761 *
6762 */
6763static int __init pci_realloc_setup_params(void)
6764{
6765	resource_alignment_param = kstrdup(resource_alignment_param,
6766					   GFP_KERNEL);
6767	disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
6768
6769	return 0;
6770}
6771pure_initcall(pci_realloc_setup_params);