1// SPDX-License-Identifier: GPL-2.0
   2/*
 
 
   3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
   4 * (C) Copyright 2007 Novell Inc.
 
 
 
   5 */
   6
   7#include <linux/pci.h>
   8#include <linux/module.h>
   9#include <linux/init.h>
  10#include <linux/device.h>
  11#include <linux/mempolicy.h>
  12#include <linux/string.h>
  13#include <linux/slab.h>
  14#include <linux/sched.h>
  15#include <linux/sched/isolation.h>
  16#include <linux/cpu.h>
  17#include <linux/pm_runtime.h>
  18#include <linux/suspend.h>
  19#include <linux/kexec.h>
  20#include <linux/of_device.h>
  21#include <linux/acpi.h>
  22#include "pci.h"
  23#include "pcie/portdrv.h"
  24
  25struct pci_dynid {
  26	struct list_head node;
  27	struct pci_device_id id;
  28};
  29
  30/**
  31 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
  32 * @drv: target pci driver
  33 * @vendor: PCI vendor ID
  34 * @device: PCI device ID
  35 * @subvendor: PCI subvendor ID
  36 * @subdevice: PCI subdevice ID
  37 * @class: PCI class
  38 * @class_mask: PCI class mask
  39 * @driver_data: private driver data
  40 *
  41 * Adds a new dynamic pci device ID to this driver and causes the
  42 * driver to probe for all devices again.  @drv must have been
  43 * registered prior to calling this function.
  44 *
  45 * CONTEXT:
  46 * Does GFP_KERNEL allocation.
  47 *
  48 * RETURNS:
  49 * 0 on success, -errno on failure.
  50 */
  51int pci_add_dynid(struct pci_driver *drv,
  52		  unsigned int vendor, unsigned int device,
  53		  unsigned int subvendor, unsigned int subdevice,
  54		  unsigned int class, unsigned int class_mask,
  55		  unsigned long driver_data)
  56{
  57	struct pci_dynid *dynid;
  58
  59	dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
  60	if (!dynid)
  61		return -ENOMEM;
  62
  63	dynid->id.vendor = vendor;
  64	dynid->id.device = device;
  65	dynid->id.subvendor = subvendor;
  66	dynid->id.subdevice = subdevice;
  67	dynid->id.class = class;
  68	dynid->id.class_mask = class_mask;
  69	dynid->id.driver_data = driver_data;
  70
  71	spin_lock(&drv->dynids.lock);
  72	list_add_tail(&dynid->node, &drv->dynids.list);
  73	spin_unlock(&drv->dynids.lock);
  74
  75	return driver_attach(&drv->driver);
  76}
  77EXPORT_SYMBOL_GPL(pci_add_dynid);
  78
  79static void pci_free_dynids(struct pci_driver *drv)
  80{
  81	struct pci_dynid *dynid, *n;
  82
  83	spin_lock(&drv->dynids.lock);
  84	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
  85		list_del(&dynid->node);
  86		kfree(dynid);
  87	}
  88	spin_unlock(&drv->dynids.lock);
  89}
  90
  91/**
  92 * store_new_id - sysfs frontend to pci_add_dynid()
  93 * @driver: target device driver
  94 * @buf: buffer for scanning device ID data
  95 * @count: input size
  96 *
  97 * Allow PCI IDs to be added to an existing driver via sysfs.
  98 */
  99static ssize_t new_id_store(struct device_driver *driver, const char *buf,
 100			    size_t count)
 101{
 102	struct pci_driver *pdrv = to_pci_driver(driver);
 103	const struct pci_device_id *ids = pdrv->id_table;
 104	u32 vendor, device, subvendor = PCI_ANY_ID,
 105		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
 106	unsigned long driver_data = 0;
 107	int fields = 0;
 108	int retval = 0;
 109
 110	fields = sscanf(buf, "%x %x %x %x %x %x %lx",
 111			&vendor, &device, &subvendor, &subdevice,
 112			&class, &class_mask, &driver_data);
 113	if (fields < 2)
 114		return -EINVAL;
 115
 116	if (fields != 7) {
 117		struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
 118		if (!pdev)
 119			return -ENOMEM;
 120
 121		pdev->vendor = vendor;
 122		pdev->device = device;
 123		pdev->subsystem_vendor = subvendor;
 124		pdev->subsystem_device = subdevice;
 125		pdev->class = class;
 126
 127		if (pci_match_id(pdrv->id_table, pdev))
 128			retval = -EEXIST;
 129
 130		kfree(pdev);
 131
 132		if (retval)
 133			return retval;
 134	}
 135
 136	/* Only accept driver_data values that match an existing id_table
 137	   entry */
 138	if (ids) {
 139		retval = -EINVAL;
 140		while (ids->vendor || ids->subvendor || ids->class_mask) {
 141			if (driver_data == ids->driver_data) {
 142				retval = 0;
 143				break;
 144			}
 145			ids++;
 146		}
 147		if (retval)	/* No match */
 148			return retval;
 149	}
 150
 151	retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
 152			       class, class_mask, driver_data);
 153	if (retval)
 154		return retval;
 155	return count;
 156}
 157static DRIVER_ATTR_WO(new_id);
 158
 159/**
 160 * store_remove_id - remove a PCI device ID from this driver
 161 * @driver: target device driver
 162 * @buf: buffer for scanning device ID data
 163 * @count: input size
 164 *
 165 * Removes a dynamic pci device ID to this driver.
 166 */
 167static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
 168			       size_t count)
 169{
 170	struct pci_dynid *dynid, *n;
 171	struct pci_driver *pdrv = to_pci_driver(driver);
 172	u32 vendor, device, subvendor = PCI_ANY_ID,
 173		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
 174	int fields = 0;
 175	size_t retval = -ENODEV;
 176
 177	fields = sscanf(buf, "%x %x %x %x %x %x",
 178			&vendor, &device, &subvendor, &subdevice,
 179			&class, &class_mask);
 180	if (fields < 2)
 181		return -EINVAL;
 182
 183	spin_lock(&pdrv->dynids.lock);
 184	list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
 185		struct pci_device_id *id = &dynid->id;
 186		if ((id->vendor == vendor) &&
 187		    (id->device == device) &&
 188		    (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
 189		    (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
 190		    !((id->class ^ class) & class_mask)) {
 191			list_del(&dynid->node);
 192			kfree(dynid);
 193			retval = count;
 194			break;
 195		}
 196	}
 197	spin_unlock(&pdrv->dynids.lock);
 198
 199	return retval;
 200}
 201static DRIVER_ATTR_WO(remove_id);
 202
 203static struct attribute *pci_drv_attrs[] = {
 204	&driver_attr_new_id.attr,
 205	&driver_attr_remove_id.attr,
 206	NULL,
 207};
 208ATTRIBUTE_GROUPS(pci_drv);
 209
 210/**
 211 * pci_match_id - See if a pci device matches a given pci_id table
 212 * @ids: array of PCI device id structures to search in
 213 * @dev: the PCI device structure to match against.
 214 *
 215 * Used by a driver to check whether a PCI device present in the
 216 * system is in its list of supported devices.  Returns the matching
 217 * pci_device_id structure or %NULL if there is no match.
 218 *
 219 * Deprecated, don't use this as it will not catch any dynamic ids
 220 * that a driver might want to check for.
 221 */
 222const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
 223					 struct pci_dev *dev)
 224{
 225	if (ids) {
 226		while (ids->vendor || ids->subvendor || ids->class_mask) {
 227			if (pci_match_one_device(ids, dev))
 228				return ids;
 229			ids++;
 230		}
 231	}
 232	return NULL;
 233}
 234EXPORT_SYMBOL(pci_match_id);
 235
 236static const struct pci_device_id pci_device_id_any = {
 237	.vendor = PCI_ANY_ID,
 238	.device = PCI_ANY_ID,
 239	.subvendor = PCI_ANY_ID,
 240	.subdevice = PCI_ANY_ID,
 241};
 242
 243/**
 244 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
 245 * @drv: the PCI driver to match against
 246 * @dev: the PCI device structure to match against
 247 *
 248 * Used by a driver to check whether a PCI device present in the
 249 * system is in its list of supported devices.  Returns the matching
 250 * pci_device_id structure or %NULL if there is no match.
 251 */
 252static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
 253						    struct pci_dev *dev)
 254{
 255	struct pci_dynid *dynid;
 256	const struct pci_device_id *found_id = NULL;
 257
 258	/* When driver_override is set, only bind to the matching driver */
 259	if (dev->driver_override && strcmp(dev->driver_override, drv->name))
 260		return NULL;
 261
 262	/* Look at the dynamic ids first, before the static ones */
 263	spin_lock(&drv->dynids.lock);
 264	list_for_each_entry(dynid, &drv->dynids.list, node) {
 265		if (pci_match_one_device(&dynid->id, dev)) {
 266			found_id = &dynid->id;
 267			break;
 268		}
 269	}
 270	spin_unlock(&drv->dynids.lock);
 271
 272	if (!found_id)
 273		found_id = pci_match_id(drv->id_table, dev);
 274
 275	/* driver_override will always match, send a dummy id */
 276	if (!found_id && dev->driver_override)
 277		found_id = &pci_device_id_any;
 278
 279	return found_id;
 280}
 281
 282struct drv_dev_and_id {
 283	struct pci_driver *drv;
 284	struct pci_dev *dev;
 285	const struct pci_device_id *id;
 286};
 287
 288static long local_pci_probe(void *_ddi)
 289{
 290	struct drv_dev_and_id *ddi = _ddi;
 291	struct pci_dev *pci_dev = ddi->dev;
 292	struct pci_driver *pci_drv = ddi->drv;
 293	struct device *dev = &pci_dev->dev;
 294	int rc;
 295
 296	/*
 297	 * Unbound PCI devices are always put in D0, regardless of
 298	 * runtime PM status.  During probe, the device is set to
 299	 * active and the usage count is incremented.  If the driver
 300	 * supports runtime PM, it should call pm_runtime_put_noidle(),
 301	 * or any other runtime PM helper function decrementing the usage
 302	 * count, in its probe routine and pm_runtime_get_noresume() in
 303	 * its remove routine.
 304	 */
 305	pm_runtime_get_sync(dev);
 306	pci_dev->driver = pci_drv;
 307	rc = pci_drv->probe(pci_dev, ddi->id);
 308	if (!rc)
 309		return rc;
 310	if (rc < 0) {
 311		pci_dev->driver = NULL;
 312		pm_runtime_put_sync(dev);
 313		return rc;
 314	}
 315	/*
 316	 * Probe function should return < 0 for failure, 0 for success
 317	 * Treat values > 0 as success, but warn.
 318	 */
 319	pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n",
 320		 rc);
 321	return 0;
 322}
 323
 324static bool pci_physfn_is_probed(struct pci_dev *dev)
 325{
 326#ifdef CONFIG_PCI_IOV
 327	return dev->is_virtfn && dev->physfn->is_probed;
 328#else
 329	return false;
 330#endif
 331}
 332
 333static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
 334			  const struct pci_device_id *id)
 335{
 336	int error, node, cpu;
 337	int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
 338	struct drv_dev_and_id ddi = { drv, dev, id };
 339
 340	/*
 341	 * Execute driver initialization on node where the device is
 342	 * attached.  This way the driver likely allocates its local memory
 343	 * on the right node.
 344	 */
 345	node = dev_to_node(&dev->dev);
 346	dev->is_probed = 1;
 347
 348	cpu_hotplug_disable();
 349
 350	/*
 351	 * Prevent nesting work_on_cpu() for the case where a Virtual Function
 352	 * device is probed from work_on_cpu() of the Physical device.
 353	 */
 354	if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
 355	    pci_physfn_is_probed(dev))
 356		cpu = nr_cpu_ids;
 357	else
 358		cpu = cpumask_any_and(cpumask_of_node(node),
 359				      housekeeping_cpumask(hk_flags));
 360
 361	if (cpu < nr_cpu_ids)
 362		error = work_on_cpu(cpu, local_pci_probe, &ddi);
 363	else
 
 
 
 
 
 
 
 
 
 
 364		error = local_pci_probe(&ddi);
 365
 366	dev->is_probed = 0;
 367	cpu_hotplug_enable();
 368	return error;
 369}
 370
 371/**
 372 * __pci_device_probe - check if a driver wants to claim a specific PCI device
 373 * @drv: driver to call to check if it wants the PCI device
 374 * @pci_dev: PCI device being probed
 375 *
 376 * returns 0 on success, else error.
 377 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
 378 */
 379static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
 380{
 381	const struct pci_device_id *id;
 382	int error = 0;
 383
 384	if (!pci_dev->driver && drv->probe) {
 385		error = -ENODEV;
 386
 387		id = pci_match_device(drv, pci_dev);
 388		if (id)
 389			error = pci_call_probe(drv, pci_dev, id);
 
 
 390	}
 391	return error;
 392}
 393
 394int __weak pcibios_alloc_irq(struct pci_dev *dev)
 395{
 396	return 0;
 397}
 398
 399void __weak pcibios_free_irq(struct pci_dev *dev)
 400{
 401}
 402
 403#ifdef CONFIG_PCI_IOV
 404static inline bool pci_device_can_probe(struct pci_dev *pdev)
 405{
 406	return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe ||
 407		pdev->driver_override);
 408}
 409#else
 410static inline bool pci_device_can_probe(struct pci_dev *pdev)
 411{
 412	return true;
 413}
 414#endif
 415
 416static int pci_device_probe(struct device *dev)
 417{
 418	int error;
 419	struct pci_dev *pci_dev = to_pci_dev(dev);
 420	struct pci_driver *drv = to_pci_driver(dev->driver);
 421
 422	if (!pci_device_can_probe(pci_dev))
 423		return -ENODEV;
 424
 425	pci_assign_irq(pci_dev);
 426
 427	error = pcibios_alloc_irq(pci_dev);
 428	if (error < 0)
 429		return error;
 430
 431	pci_dev_get(pci_dev);
 432	error = __pci_device_probe(drv, pci_dev);
 433	if (error) {
 434		pcibios_free_irq(pci_dev);
 435		pci_dev_put(pci_dev);
 436	}
 437
 438	return error;
 439}
 440
 441static int pci_device_remove(struct device *dev)
 442{
 443	struct pci_dev *pci_dev = to_pci_dev(dev);
 444	struct pci_driver *drv = pci_dev->driver;
 445
 446	if (drv) {
 447		if (drv->remove) {
 448			pm_runtime_get_sync(dev);
 449			drv->remove(pci_dev);
 450			pm_runtime_put_noidle(dev);
 451		}
 452		pcibios_free_irq(pci_dev);
 453		pci_dev->driver = NULL;
 454		pci_iov_remove(pci_dev);
 455	}
 456
 457	/* Undo the runtime PM settings in local_pci_probe() */
 458	pm_runtime_put_sync(dev);
 459
 460	/*
 461	 * If the device is still on, set the power state as "unknown",
 462	 * since it might change by the next time we load the driver.
 463	 */
 464	if (pci_dev->current_state == PCI_D0)
 465		pci_dev->current_state = PCI_UNKNOWN;
 466
 467	/*
 468	 * We would love to complain here if pci_dev->is_enabled is set, that
 469	 * the driver should have called pci_disable_device(), but the
 470	 * unfortunate fact is there are too many odd BIOS and bridge setups
 471	 * that don't like drivers doing that all of the time.
 472	 * Oh well, we can dream of sane hardware when we sleep, no matter how
 473	 * horrible the crap we have to deal with is when we are awake...
 474	 */
 475
 476	pci_dev_put(pci_dev);
 477	return 0;
 478}
 479
 480static void pci_device_shutdown(struct device *dev)
 481{
 482	struct pci_dev *pci_dev = to_pci_dev(dev);
 483	struct pci_driver *drv = pci_dev->driver;
 484
 485	pm_runtime_resume(dev);
 486
 487	if (drv && drv->shutdown)
 488		drv->shutdown(pci_dev);
 
 
 489
 490	/*
 491	 * If this is a kexec reboot, turn off Bus Master bit on the
 492	 * device to tell it to not continue to do DMA. Don't touch
 493	 * devices in D3cold or unknown states.
 494	 * If it is not a kexec reboot, firmware will hit the PCI
 495	 * devices with big hammer and stop their DMA any way.
 496	 */
 497	if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
 498		pci_clear_master(pci_dev);
 499}
 500
 501#ifdef CONFIG_PM
 502
 503/* Auxiliary functions used for system resume and run-time resume. */
 504
 505/**
 506 * pci_restore_standard_config - restore standard config registers of PCI device
 507 * @pci_dev: PCI device to handle
 508 */
 509static int pci_restore_standard_config(struct pci_dev *pci_dev)
 510{
 511	pci_update_current_state(pci_dev, PCI_UNKNOWN);
 512
 513	if (pci_dev->current_state != PCI_D0) {
 514		int error = pci_set_power_state(pci_dev, PCI_D0);
 515		if (error)
 516			return error;
 517	}
 518
 519	pci_restore_state(pci_dev);
 520	pci_pme_restore(pci_dev);
 521	return 0;
 522}
 523
 524static void pci_pm_default_resume(struct pci_dev *pci_dev)
 525{
 526	pci_fixup_device(pci_fixup_resume, pci_dev);
 527	pci_enable_wake(pci_dev, PCI_D0, false);
 528}
 529
 530#endif
 531
 532#ifdef CONFIG_PM_SLEEP
 533
 534static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
 535{
 536	pci_power_up(pci_dev);
 537	pci_update_current_state(pci_dev, PCI_D0);
 538	pci_restore_state(pci_dev);
 539	pci_pme_restore(pci_dev);
 540}
 541
 542/*
 543 * Default "suspend" method for devices that have no driver provided suspend,
 544 * or not even a driver at all (second part).
 545 */
 546static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
 547{
 548	/*
 549	 * mark its power state as "unknown", since we don't know if
 550	 * e.g. the BIOS will change its device state when we suspend.
 551	 */
 552	if (pci_dev->current_state == PCI_D0)
 553		pci_dev->current_state = PCI_UNKNOWN;
 554}
 555
 556/*
 557 * Default "resume" method for devices that have no driver provided resume,
 558 * or not even a driver at all (second part).
 559 */
 560static int pci_pm_reenable_device(struct pci_dev *pci_dev)
 561{
 562	int retval;
 563
 564	/* if the device was enabled before suspend, reenable */
 565	retval = pci_reenable_device(pci_dev);
 566	/*
 567	 * if the device was busmaster before the suspend, make it busmaster
 568	 * again
 569	 */
 570	if (pci_dev->is_busmaster)
 571		pci_set_master(pci_dev);
 572
 573	return retval;
 574}
 575
 576static int pci_legacy_suspend(struct device *dev, pm_message_t state)
 577{
 578	struct pci_dev *pci_dev = to_pci_dev(dev);
 579	struct pci_driver *drv = pci_dev->driver;
 580
 581	if (drv && drv->suspend) {
 582		pci_power_t prev = pci_dev->current_state;
 583		int error;
 584
 585		error = drv->suspend(pci_dev, state);
 586		suspend_report_result(drv->suspend, error);
 587		if (error)
 588			return error;
 589
 590		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
 591		    && pci_dev->current_state != PCI_UNKNOWN) {
 592			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
 593				      "PCI PM: Device state not saved by %pS\n",
 594				      drv->suspend);
 595		}
 596	}
 597
 598	pci_fixup_device(pci_fixup_suspend, pci_dev);
 599
 600	return 0;
 601}
 602
 603static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
 604{
 605	struct pci_dev *pci_dev = to_pci_dev(dev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 606
 607	if (!pci_dev->state_saved)
 608		pci_save_state(pci_dev);
 609
 610	pci_pm_set_unknown_state(pci_dev);
 611
 
 612	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
 613
 614	return 0;
 615}
 616
 
 
 
 
 
 
 
 
 
 617static int pci_legacy_resume(struct device *dev)
 618{
 619	struct pci_dev *pci_dev = to_pci_dev(dev);
 620	struct pci_driver *drv = pci_dev->driver;
 621
 622	pci_fixup_device(pci_fixup_resume, pci_dev);
 623
 624	return drv && drv->resume ?
 625			drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
 626}
 627
 628/* Auxiliary functions used by the new power management framework */
 629
 
 
 
 
 
 
 
 
 630static void pci_pm_default_suspend(struct pci_dev *pci_dev)
 631{
 632	/* Disable non-bridge devices without PM support */
 633	if (!pci_has_subordinate(pci_dev))
 634		pci_disable_enabled_device(pci_dev);
 635}
 636
 637static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
 638{
 639	struct pci_driver *drv = pci_dev->driver;
 640	bool ret = drv && (drv->suspend || drv->resume);
 
 641
 642	/*
 643	 * Legacy PM support is used by default, so warn if the new framework is
 644	 * supported as well.  Drivers are supposed to support either the
 645	 * former, or the latter, but not both at the same time.
 646	 */
 647	pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n",
 648		 pci_dev->vendor, pci_dev->device);
 649
 650	return ret;
 651}
 652
 653/* New power management framework */
 654
 655static int pci_pm_prepare(struct device *dev)
 656{
 657	struct pci_dev *pci_dev = to_pci_dev(dev);
 658	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 659
 660	if (pm && pm->prepare) {
 661		int error = pm->prepare(dev);
 662		if (error < 0)
 663			return error;
 664
 665		if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
 666			return 0;
 667	}
 668	if (pci_dev_need_resume(pci_dev))
 669		return 0;
 670
 671	/*
 672	 * The PME setting needs to be adjusted here in case the direct-complete
 673	 * optimization is used with respect to this device.
 674	 */
 675	pci_dev_adjust_pme(pci_dev);
 676	return 1;
 
 
 
 
 
 
 
 677}
 678
 679static void pci_pm_complete(struct device *dev)
 680{
 681	struct pci_dev *pci_dev = to_pci_dev(dev);
 682
 683	pci_dev_complete_resume(pci_dev);
 684	pm_generic_complete(dev);
 685
 686	/* Resume device if platform firmware has put it in reset-power-on */
 687	if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
 688		pci_power_t pre_sleep_state = pci_dev->current_state;
 689
 690		pci_refresh_power_state(pci_dev);
 691		/*
 692		 * On platforms with ACPI this check may also trigger for
 693		 * devices sharing power resources if one of those power
 694		 * resources has been activated as a result of a change of the
 695		 * power state of another device sharing it.  However, in that
 696		 * case it is also better to resume the device, in general.
 697		 */
 698		if (pci_dev->current_state < pre_sleep_state)
 699			pm_request_resume(dev);
 700	}
 701}
 702
 703#else /* !CONFIG_PM_SLEEP */
 704
 705#define pci_pm_prepare	NULL
 706#define pci_pm_complete	NULL
 707
 708#endif /* !CONFIG_PM_SLEEP */
 709
 710#ifdef CONFIG_SUSPEND
 711static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
 712{
 713	/*
 714	 * Some BIOSes forget to clear Root PME Status bits after system
 715	 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
 716	 * Clear those bits now just in case (shouldn't hurt).
 717	 */
 718	if (pci_is_pcie(pci_dev) &&
 719	    (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
 720	     pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
 721		pcie_clear_root_pme_status(pci_dev);
 722}
 723
 724static int pci_pm_suspend(struct device *dev)
 725{
 726	struct pci_dev *pci_dev = to_pci_dev(dev);
 727	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 728
 729	pci_dev->skip_bus_pm = false;
 730
 731	if (pci_has_legacy_pm_support(pci_dev))
 732		return pci_legacy_suspend(dev, PMSG_SUSPEND);
 733
 734	if (!pm) {
 735		pci_pm_default_suspend(pci_dev);
 736		return 0;
 737	}
 738
 739	/*
 740	 * PCI devices suspended at run time may need to be resumed at this
 741	 * point, because in general it may be necessary to reconfigure them for
 742	 * system suspend.  Namely, if the device is expected to wake up the
 743	 * system from the sleep state, it may have to be reconfigured for this
 744	 * purpose, or if the device is not expected to wake up the system from
 745	 * the sleep state, it should be prevented from signaling wakeup events
 746	 * going forward.
 747	 *
 748	 * Also if the driver of the device does not indicate that its system
 749	 * suspend callbacks can cope with runtime-suspended devices, it is
 750	 * better to resume the device from runtime suspend here.
 751	 */
 752	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
 753	    pci_dev_need_resume(pci_dev)) {
 754		pm_runtime_resume(dev);
 755		pci_dev->state_saved = false;
 756	} else {
 757		pci_dev_adjust_pme(pci_dev);
 758	}
 759
 
 760	if (pm->suspend) {
 761		pci_power_t prev = pci_dev->current_state;
 762		int error;
 763
 764		error = pm->suspend(dev);
 765		suspend_report_result(pm->suspend, error);
 766		if (error)
 767			return error;
 768
 769		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
 770		    && pci_dev->current_state != PCI_UNKNOWN) {
 771			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
 772				      "PCI PM: State of device not saved by %pS\n",
 773				      pm->suspend);
 774		}
 775	}
 776
 777	return 0;
 778}
 779
 780static int pci_pm_suspend_late(struct device *dev)
 781{
 782	if (dev_pm_skip_suspend(dev))
 783		return 0;
 784
 785	pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
 786
 787	return pm_generic_suspend_late(dev);
 788}
 789
 790static int pci_pm_suspend_noirq(struct device *dev)
 791{
 792	struct pci_dev *pci_dev = to_pci_dev(dev);
 793	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 794
 795	if (dev_pm_skip_suspend(dev))
 796		return 0;
 797
 798	if (pci_has_legacy_pm_support(pci_dev))
 799		return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
 800
 801	if (!pm) {
 802		pci_save_state(pci_dev);
 803		goto Fixup;
 804	}
 805
 806	if (pm->suspend_noirq) {
 807		pci_power_t prev = pci_dev->current_state;
 808		int error;
 809
 810		error = pm->suspend_noirq(dev);
 811		suspend_report_result(pm->suspend_noirq, error);
 812		if (error)
 813			return error;
 814
 815		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
 816		    && pci_dev->current_state != PCI_UNKNOWN) {
 817			pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
 818				      "PCI PM: State of device not saved by %pS\n",
 819				      pm->suspend_noirq);
 820			goto Fixup;
 821		}
 822	}
 823
 824	if (pci_dev->skip_bus_pm) {
 825		/*
 826		 * Either the device is a bridge with a child in D0 below it, or
 827		 * the function is running for the second time in a row without
 828		 * going through full resume, which is possible only during
 829		 * suspend-to-idle in a spurious wakeup case.  The device should
 830		 * be in D0 at this point, but if it is a bridge, it may be
 831		 * necessary to save its state.
 832		 */
 833		if (!pci_dev->state_saved)
 834			pci_save_state(pci_dev);
 835	} else if (!pci_dev->state_saved) {
 836		pci_save_state(pci_dev);
 837		if (pci_power_manageable(pci_dev))
 838			pci_prepare_to_sleep(pci_dev);
 839	}
 840
 841	pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n",
 842		pci_power_name(pci_dev->current_state));
 843
 844	if (pci_dev->current_state == PCI_D0) {
 845		pci_dev->skip_bus_pm = true;
 846		/*
 847		 * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any
 848		 * downstream device is in D0, so avoid changing the power state
 849		 * of the parent bridge by setting the skip_bus_pm flag for it.
 850		 */
 851		if (pci_dev->bus->self)
 852			pci_dev->bus->self->skip_bus_pm = true;
 853	}
 854
 855	if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) {
 856		pci_dbg(pci_dev, "PCI PM: Skipped\n");
 857		goto Fixup;
 858	}
 859
 860	pci_pm_set_unknown_state(pci_dev);
 861
 862	/*
 863	 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
 864	 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
 865	 * hasn't been quiesced and tries to turn it off.  If the controller
 866	 * is already in D3, this can hang or cause memory corruption.
 867	 *
 868	 * Since the value of the COMMAND register doesn't matter once the
 869	 * device has been suspended, we can safely set it to 0 here.
 870	 */
 871	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
 872		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
 873
 874Fixup:
 875	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
 876
 877	/*
 878	 * If the target system sleep state is suspend-to-idle, it is sufficient
 879	 * to check whether or not the device's wakeup settings are good for
 880	 * runtime PM.  Otherwise, the pm_resume_via_firmware() check will cause
 881	 * pci_pm_complete() to take care of fixing up the device's state
 882	 * anyway, if need be.
 883	 */
 884	if (device_can_wakeup(dev) && !device_may_wakeup(dev))
 885		dev->power.may_skip_resume = false;
 886
 887	return 0;
 888}
 889
 890static int pci_pm_resume_noirq(struct device *dev)
 891{
 892	struct pci_dev *pci_dev = to_pci_dev(dev);
 893	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 894	pci_power_t prev_state = pci_dev->current_state;
 895	bool skip_bus_pm = pci_dev->skip_bus_pm;
 896
 897	if (dev_pm_skip_resume(dev))
 898		return 0;
 899
 900	/*
 901	 * In the suspend-to-idle case, devices left in D0 during suspend will
 902	 * stay in D0, so it is not necessary to restore or update their
 903	 * configuration here and attempting to put them into D0 again is
 904	 * pointless, so avoid doing that.
 905	 */
 906	if (!(skip_bus_pm && pm_suspend_no_platform()))
 907		pci_pm_default_resume_early(pci_dev);
 908
 909	pci_fixup_device(pci_fixup_resume_early, pci_dev);
 910	pcie_pme_root_status_cleanup(pci_dev);
 911
 912	if (!skip_bus_pm && prev_state == PCI_D3cold)
 913		pci_bridge_wait_for_secondary_bus(pci_dev);
 914
 915	if (pci_has_legacy_pm_support(pci_dev))
 916		return 0;
 917
 918	if (pm && pm->resume_noirq)
 919		return pm->resume_noirq(dev);
 920
 921	return 0;
 922}
 923
 924static int pci_pm_resume_early(struct device *dev)
 925{
 926	if (dev_pm_skip_resume(dev))
 927		return 0;
 928
 929	return pm_generic_resume_early(dev);
 930}
 931
 932static int pci_pm_resume(struct device *dev)
 933{
 934	struct pci_dev *pci_dev = to_pci_dev(dev);
 935	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
 936
 937	/*
 938	 * This is necessary for the suspend error path in which resume is
 939	 * called without restoring the standard config registers of the device.
 940	 */
 941	if (pci_dev->state_saved)
 942		pci_restore_standard_config(pci_dev);
 943
 944	if (pci_has_legacy_pm_support(pci_dev))
 945		return pci_legacy_resume(dev);
 946
 947	pci_pm_default_resume(pci_dev);
 948
 949	if (pm) {
 950		if (pm->resume)
 951			return pm->resume(dev);
 952	} else {
 953		pci_pm_reenable_device(pci_dev);
 954	}
 955
 956	return 0;
 957}
 958
 959#else /* !CONFIG_SUSPEND */
 960
 961#define pci_pm_suspend		NULL
 962#define pci_pm_suspend_late	NULL
 963#define pci_pm_suspend_noirq	NULL
 964#define pci_pm_resume		NULL
 965#define pci_pm_resume_early	NULL
 966#define pci_pm_resume_noirq	NULL
 967
 968#endif /* !CONFIG_SUSPEND */
 969
 970#ifdef CONFIG_HIBERNATE_CALLBACKS
 971
 
 972/*
 973 * pcibios_pm_ops - provide arch-specific hooks when a PCI device is doing
 974 * a hibernate transition
 975 */
 976struct dev_pm_ops __weak pcibios_pm_ops;
 977
 978static int pci_pm_freeze(struct device *dev)
 979{
 980	struct pci_dev *pci_dev = to_pci_dev(dev);
 981	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 982
 983	if (pci_has_legacy_pm_support(pci_dev))
 984		return pci_legacy_suspend(dev, PMSG_FREEZE);
 985
 986	if (!pm) {
 987		pci_pm_default_suspend(pci_dev);
 988		return 0;
 989	}
 990
 991	/*
 992	 * Resume all runtime-suspended devices before creating a snapshot
 993	 * image of system memory, because the restore kernel generally cannot
 994	 * be expected to always handle them consistently and they need to be
 995	 * put into the runtime-active metastate during system resume anyway,
 996	 * so it is better to ensure that the state saved in the image will be
 997	 * always consistent with that.
 998	 */
 999	pm_runtime_resume(dev);
1000	pci_dev->state_saved = false;
1001
 
1002	if (pm->freeze) {
1003		int error;
1004
1005		error = pm->freeze(dev);
1006		suspend_report_result(pm->freeze, error);
1007		if (error)
1008			return error;
1009	}
1010
 
 
 
1011	return 0;
1012}
1013
1014static int pci_pm_freeze_noirq(struct device *dev)
1015{
1016	struct pci_dev *pci_dev = to_pci_dev(dev);
1017	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1018
1019	if (pci_has_legacy_pm_support(pci_dev))
1020		return pci_legacy_suspend_late(dev, PMSG_FREEZE);
1021
1022	if (pm && pm->freeze_noirq) {
1023		int error;
1024
1025		error = pm->freeze_noirq(dev);
1026		suspend_report_result(pm->freeze_noirq, error);
1027		if (error)
1028			return error;
1029	}
1030
1031	if (!pci_dev->state_saved)
1032		pci_save_state(pci_dev);
1033
1034	pci_pm_set_unknown_state(pci_dev);
1035
1036	if (pcibios_pm_ops.freeze_noirq)
1037		return pcibios_pm_ops.freeze_noirq(dev);
1038
1039	return 0;
1040}
1041
1042static int pci_pm_thaw_noirq(struct device *dev)
1043{
1044	struct pci_dev *pci_dev = to_pci_dev(dev);
1045	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1046	int error;
1047
1048	if (pcibios_pm_ops.thaw_noirq) {
1049		error = pcibios_pm_ops.thaw_noirq(dev);
1050		if (error)
1051			return error;
1052	}
1053
1054	/*
1055	 * The pm->thaw_noirq() callback assumes the device has been
1056	 * returned to D0 and its config state has been restored.
1057	 *
1058	 * In addition, pci_restore_state() restores MSI-X state in MMIO
1059	 * space, which requires the device to be in D0, so return it to D0
1060	 * in case the driver's "freeze" callbacks put it into a low-power
1061	 * state.
1062	 */
1063	pci_set_power_state(pci_dev, PCI_D0);
1064	pci_restore_state(pci_dev);
1065
1066	if (pci_has_legacy_pm_support(pci_dev))
1067		return 0;
1068
1069	if (pm && pm->thaw_noirq)
1070		return pm->thaw_noirq(dev);
1071
1072	return 0;
 
 
 
1073}
1074
1075static int pci_pm_thaw(struct device *dev)
1076{
1077	struct pci_dev *pci_dev = to_pci_dev(dev);
1078	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1079	int error = 0;
1080
 
 
 
 
 
 
1081	if (pci_has_legacy_pm_support(pci_dev))
1082		return pci_legacy_resume(dev);
1083
1084	if (pm) {
1085		if (pm->thaw)
1086			error = pm->thaw(dev);
1087	} else {
1088		pci_pm_reenable_device(pci_dev);
1089	}
1090
1091	pci_dev->state_saved = false;
1092
1093	return error;
1094}
1095
1096static int pci_pm_poweroff(struct device *dev)
1097{
1098	struct pci_dev *pci_dev = to_pci_dev(dev);
1099	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1100
1101	if (pci_has_legacy_pm_support(pci_dev))
1102		return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1103
1104	if (!pm) {
1105		pci_pm_default_suspend(pci_dev);
1106		return 0;
1107	}
1108
1109	/* The reason to do that is the same as in pci_pm_suspend(). */
1110	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1111	    pci_dev_need_resume(pci_dev)) {
1112		pm_runtime_resume(dev);
1113		pci_dev->state_saved = false;
1114	} else {
1115		pci_dev_adjust_pme(pci_dev);
1116	}
1117
 
1118	if (pm->poweroff) {
1119		int error;
1120
1121		error = pm->poweroff(dev);
1122		suspend_report_result(pm->poweroff, error);
1123		if (error)
1124			return error;
1125	}
1126
1127	return 0;
1128}
1129
1130static int pci_pm_poweroff_late(struct device *dev)
1131{
1132	if (dev_pm_skip_suspend(dev))
1133		return 0;
1134
1135	pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
 
1136
1137	return pm_generic_poweroff_late(dev);
1138}
1139
1140static int pci_pm_poweroff_noirq(struct device *dev)
1141{
1142	struct pci_dev *pci_dev = to_pci_dev(dev);
1143	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1144
1145	if (dev_pm_skip_suspend(dev))
1146		return 0;
1147
1148	if (pci_has_legacy_pm_support(pci_dev))
1149		return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
1150
1151	if (!pm) {
1152		pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1153		return 0;
1154	}
1155
1156	if (pm->poweroff_noirq) {
1157		int error;
1158
1159		error = pm->poweroff_noirq(dev);
1160		suspend_report_result(pm->poweroff_noirq, error);
1161		if (error)
1162			return error;
1163	}
1164
1165	if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1166		pci_prepare_to_sleep(pci_dev);
1167
1168	/*
1169	 * The reason for doing this here is the same as for the analogous code
1170	 * in pci_pm_suspend_noirq().
1171	 */
1172	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1173		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1174
1175	pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1176
1177	if (pcibios_pm_ops.poweroff_noirq)
1178		return pcibios_pm_ops.poweroff_noirq(dev);
1179
1180	return 0;
1181}
1182
1183static int pci_pm_restore_noirq(struct device *dev)
1184{
1185	struct pci_dev *pci_dev = to_pci_dev(dev);
1186	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1187	int error;
1188
1189	if (pcibios_pm_ops.restore_noirq) {
1190		error = pcibios_pm_ops.restore_noirq(dev);
1191		if (error)
1192			return error;
1193	}
1194
1195	pci_pm_default_resume_early(pci_dev);
1196	pci_fixup_device(pci_fixup_resume_early, pci_dev);
1197
1198	if (pci_has_legacy_pm_support(pci_dev))
1199		return 0;
1200
1201	if (pm && pm->restore_noirq)
1202		return pm->restore_noirq(dev);
1203
1204	return 0;
1205}
1206
1207static int pci_pm_restore(struct device *dev)
1208{
1209	struct pci_dev *pci_dev = to_pci_dev(dev);
1210	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
 
 
 
 
 
 
1211
1212	/*
1213	 * This is necessary for the hibernation error path in which restore is
1214	 * called without restoring the standard config registers of the device.
1215	 */
1216	if (pci_dev->state_saved)
1217		pci_restore_standard_config(pci_dev);
1218
1219	if (pci_has_legacy_pm_support(pci_dev))
1220		return pci_legacy_resume(dev);
1221
1222	pci_pm_default_resume(pci_dev);
1223
1224	if (pm) {
1225		if (pm->restore)
1226			return pm->restore(dev);
1227	} else {
1228		pci_pm_reenable_device(pci_dev);
1229	}
1230
1231	return 0;
1232}
1233
1234#else /* !CONFIG_HIBERNATE_CALLBACKS */
1235
1236#define pci_pm_freeze		NULL
1237#define pci_pm_freeze_noirq	NULL
1238#define pci_pm_thaw		NULL
1239#define pci_pm_thaw_noirq	NULL
1240#define pci_pm_poweroff		NULL
1241#define pci_pm_poweroff_late	NULL
1242#define pci_pm_poweroff_noirq	NULL
1243#define pci_pm_restore		NULL
1244#define pci_pm_restore_noirq	NULL
1245
1246#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1247
1248#ifdef CONFIG_PM
1249
1250static int pci_pm_runtime_suspend(struct device *dev)
1251{
1252	struct pci_dev *pci_dev = to_pci_dev(dev);
1253	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1254	pci_power_t prev = pci_dev->current_state;
1255	int error;
1256
1257	/*
1258	 * If pci_dev->driver is not set (unbound), we leave the device in D0,
1259	 * but it may go to D3cold when the bridge above it runtime suspends.
1260	 * Save its config space in case that happens.
1261	 */
1262	if (!pci_dev->driver) {
1263		pci_save_state(pci_dev);
1264		return 0;
1265	}
 
 
1266
1267	pci_dev->state_saved = false;
1268	if (pm && pm->runtime_suspend) {
1269		error = pm->runtime_suspend(dev);
1270		/*
1271		 * -EBUSY and -EAGAIN is used to request the runtime PM core
1272		 * to schedule a new suspend, so log the event only with debug
1273		 * log level.
1274		 */
1275		if (error == -EBUSY || error == -EAGAIN) {
1276			pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n",
1277				pm->runtime_suspend, error);
1278			return error;
1279		} else if (error) {
1280			pci_err(pci_dev, "can't suspend (%ps returned %d)\n",
1281				pm->runtime_suspend, error);
1282			return error;
1283		}
1284	}
1285
1286	pci_fixup_device(pci_fixup_suspend, pci_dev);
1287
1288	if (pm && pm->runtime_suspend
1289	    && !pci_dev->state_saved && pci_dev->current_state != PCI_D0
1290	    && pci_dev->current_state != PCI_UNKNOWN) {
1291		pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
1292			      "PCI PM: State of device not saved by %pS\n",
1293			      pm->runtime_suspend);
1294		return 0;
1295	}
1296
1297	if (!pci_dev->state_saved) {
1298		pci_save_state(pci_dev);
1299		pci_finish_runtime_suspend(pci_dev);
1300	}
1301
1302	return 0;
1303}
1304
1305static int pci_pm_runtime_resume(struct device *dev)
1306{
 
1307	struct pci_dev *pci_dev = to_pci_dev(dev);
1308	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1309	pci_power_t prev_state = pci_dev->current_state;
1310	int error = 0;
1311
1312	/*
1313	 * Restoring config space is necessary even if the device is not bound
1314	 * to a driver because although we left it in D0, it may have gone to
1315	 * D3cold when the bridge above it runtime suspended.
1316	 */
1317	pci_restore_standard_config(pci_dev);
1318
1319	if (!pci_dev->driver)
1320		return 0;
1321
1322	pci_fixup_device(pci_fixup_resume_early, pci_dev);
1323	pci_pm_default_resume(pci_dev);
1324
1325	if (prev_state == PCI_D3cold)
1326		pci_bridge_wait_for_secondary_bus(pci_dev);
 
 
1327
1328	if (pm && pm->runtime_resume)
1329		error = pm->runtime_resume(dev);
1330
1331	pci_dev->runtime_d3cold = false;
1332
1333	return error;
1334}
1335
1336static int pci_pm_runtime_idle(struct device *dev)
1337{
1338	struct pci_dev *pci_dev = to_pci_dev(dev);
1339	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
1340
1341	/*
1342	 * If pci_dev->driver is not set (unbound), the device should
1343	 * always remain in D0 regardless of the runtime PM status
1344	 */
1345	if (!pci_dev->driver)
1346		return 0;
1347
1348	if (!pm)
1349		return -ENOSYS;
1350
1351	if (pm->runtime_idle)
1352		return pm->runtime_idle(dev);
1353
1354	return 0;
1355}
1356
1357static const struct dev_pm_ops pci_dev_pm_ops = {
1358	.prepare = pci_pm_prepare,
1359	.complete = pci_pm_complete,
1360	.suspend = pci_pm_suspend,
1361	.suspend_late = pci_pm_suspend_late,
1362	.resume = pci_pm_resume,
1363	.resume_early = pci_pm_resume_early,
1364	.freeze = pci_pm_freeze,
1365	.thaw = pci_pm_thaw,
1366	.poweroff = pci_pm_poweroff,
1367	.poweroff_late = pci_pm_poweroff_late,
1368	.restore = pci_pm_restore,
1369	.suspend_noirq = pci_pm_suspend_noirq,
1370	.resume_noirq = pci_pm_resume_noirq,
1371	.freeze_noirq = pci_pm_freeze_noirq,
1372	.thaw_noirq = pci_pm_thaw_noirq,
1373	.poweroff_noirq = pci_pm_poweroff_noirq,
1374	.restore_noirq = pci_pm_restore_noirq,
1375	.runtime_suspend = pci_pm_runtime_suspend,
1376	.runtime_resume = pci_pm_runtime_resume,
1377	.runtime_idle = pci_pm_runtime_idle,
1378};
1379
1380#define PCI_PM_OPS_PTR	(&pci_dev_pm_ops)
1381
1382#else /* !CONFIG_PM */
1383
1384#define pci_pm_runtime_suspend	NULL
1385#define pci_pm_runtime_resume	NULL
1386#define pci_pm_runtime_idle	NULL
1387
1388#define PCI_PM_OPS_PTR	NULL
1389
1390#endif /* !CONFIG_PM */
1391
1392/**
1393 * __pci_register_driver - register a new pci driver
1394 * @drv: the driver structure to register
1395 * @owner: owner module of drv
1396 * @mod_name: module name string
1397 *
1398 * Adds the driver structure to the list of registered drivers.
1399 * Returns a negative value on error, otherwise 0.
1400 * If no error occurred, the driver remains registered even if
1401 * no device was claimed during registration.
1402 */
1403int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1404			  const char *mod_name)
1405{
1406	/* initialize common driver fields */
1407	drv->driver.name = drv->name;
1408	drv->driver.bus = &pci_bus_type;
1409	drv->driver.owner = owner;
1410	drv->driver.mod_name = mod_name;
1411	drv->driver.groups = drv->groups;
1412
1413	spin_lock_init(&drv->dynids.lock);
1414	INIT_LIST_HEAD(&drv->dynids.list);
1415
1416	/* register with core */
1417	return driver_register(&drv->driver);
1418}
1419EXPORT_SYMBOL(__pci_register_driver);
1420
1421/**
1422 * pci_unregister_driver - unregister a pci driver
1423 * @drv: the driver structure to unregister
1424 *
1425 * Deletes the driver structure from the list of registered PCI drivers,
1426 * gives it a chance to clean up by calling its remove() function for
1427 * each device it was responsible for, and marks those devices as
1428 * driverless.
1429 */
1430
1431void pci_unregister_driver(struct pci_driver *drv)
1432{
1433	driver_unregister(&drv->driver);
1434	pci_free_dynids(drv);
1435}
1436EXPORT_SYMBOL(pci_unregister_driver);
1437
1438static struct pci_driver pci_compat_driver = {
1439	.name = "compat"
1440};
1441
1442/**
1443 * pci_dev_driver - get the pci_driver of a device
1444 * @dev: the device to query
1445 *
1446 * Returns the appropriate pci_driver structure or %NULL if there is no
1447 * registered driver for the device.
1448 */
1449struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1450{
1451	if (dev->driver)
1452		return dev->driver;
1453	else {
1454		int i;
1455		for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1456			if (dev->resource[i].flags & IORESOURCE_BUSY)
1457				return &pci_compat_driver;
1458	}
1459	return NULL;
1460}
1461EXPORT_SYMBOL(pci_dev_driver);
1462
1463/**
1464 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1465 * @dev: the PCI device structure to match against
1466 * @drv: the device driver to search for matching PCI device id structures
1467 *
1468 * Used by a driver to check whether a PCI device present in the
1469 * system is in its list of supported devices. Returns the matching
1470 * pci_device_id structure or %NULL if there is no match.
1471 */
1472static int pci_bus_match(struct device *dev, struct device_driver *drv)
1473{
1474	struct pci_dev *pci_dev = to_pci_dev(dev);
1475	struct pci_driver *pci_drv;
1476	const struct pci_device_id *found_id;
1477
1478	if (!pci_dev->match_driver)
1479		return 0;
1480
1481	pci_drv = to_pci_driver(drv);
1482	found_id = pci_match_device(pci_drv, pci_dev);
1483	if (found_id)
1484		return 1;
1485
1486	return 0;
1487}
1488
1489/**
1490 * pci_dev_get - increments the reference count of the pci device structure
1491 * @dev: the device being referenced
1492 *
1493 * Each live reference to a device should be refcounted.
1494 *
1495 * Drivers for PCI devices should normally record such references in
1496 * their probe() methods, when they bind to a device, and release
1497 * them by calling pci_dev_put(), in their disconnect() methods.
1498 *
1499 * A pointer to the device with the incremented reference counter is returned.
1500 */
1501struct pci_dev *pci_dev_get(struct pci_dev *dev)
1502{
1503	if (dev)
1504		get_device(&dev->dev);
1505	return dev;
1506}
1507EXPORT_SYMBOL(pci_dev_get);
1508
1509/**
1510 * pci_dev_put - release a use of the pci device structure
1511 * @dev: device that's been disconnected
1512 *
1513 * Must be called when a user of a device is finished with it.  When the last
1514 * user of the device calls this function, the memory of the device is freed.
1515 */
1516void pci_dev_put(struct pci_dev *dev)
1517{
1518	if (dev)
1519		put_device(&dev->dev);
1520}
1521EXPORT_SYMBOL(pci_dev_put);
1522
1523static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1524{
1525	struct pci_dev *pdev;
1526
1527	if (!dev)
1528		return -ENODEV;
1529
1530	pdev = to_pci_dev(dev);
1531
1532	if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1533		return -ENOMEM;
1534
1535	if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1536		return -ENOMEM;
1537
1538	if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1539			   pdev->subsystem_device))
1540		return -ENOMEM;
1541
1542	if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1543		return -ENOMEM;
1544
1545	if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1546			   pdev->vendor, pdev->device,
1547			   pdev->subsystem_vendor, pdev->subsystem_device,
1548			   (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1549			   (u8)(pdev->class)))
1550		return -ENOMEM;
1551
1552	return 0;
1553}
1554
1555#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH)
1556/**
1557 * pci_uevent_ers - emit a uevent during recovery path of PCI device
1558 * @pdev: PCI device undergoing error recovery
1559 * @err_type: type of error event
1560 */
1561void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1562{
1563	int idx = 0;
1564	char *envp[3];
1565
1566	switch (err_type) {
1567	case PCI_ERS_RESULT_NONE:
1568	case PCI_ERS_RESULT_CAN_RECOVER:
1569		envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1570		envp[idx++] = "DEVICE_ONLINE=0";
1571		break;
1572	case PCI_ERS_RESULT_RECOVERED:
1573		envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1574		envp[idx++] = "DEVICE_ONLINE=1";
1575		break;
1576	case PCI_ERS_RESULT_DISCONNECT:
1577		envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1578		envp[idx++] = "DEVICE_ONLINE=0";
1579		break;
1580	default:
1581		break;
1582	}
1583
1584	if (idx > 0) {
1585		envp[idx++] = NULL;
1586		kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1587	}
1588}
1589#endif
1590
1591static int pci_bus_num_vf(struct device *dev)
1592{
1593	return pci_num_vf(to_pci_dev(dev));
1594}
1595
1596/**
1597 * pci_dma_configure - Setup DMA configuration
1598 * @dev: ptr to dev structure
1599 *
1600 * Function to update PCI devices's DMA configuration using the same
1601 * info from the OF node or ACPI node of host bridge's parent (if any).
1602 */
1603static int pci_dma_configure(struct device *dev)
1604{
1605	struct device *bridge;
1606	int ret = 0;
1607
1608	bridge = pci_get_host_bridge_device(to_pci_dev(dev));
1609
1610	if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
1611	    bridge->parent->of_node) {
1612		ret = of_dma_configure(dev, bridge->parent->of_node, true);
1613	} else if (has_acpi_companion(bridge)) {
1614		struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
1615
1616		ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev));
1617	}
1618
1619	pci_put_host_bridge_device(bridge);
1620	return ret;
1621}
1622
1623struct bus_type pci_bus_type = {
1624	.name		= "pci",
1625	.match		= pci_bus_match,
1626	.uevent		= pci_uevent,
1627	.probe		= pci_device_probe,
1628	.remove		= pci_device_remove,
1629	.shutdown	= pci_device_shutdown,
1630	.dev_groups	= pci_dev_groups,
1631	.bus_groups	= pci_bus_groups,
1632	.drv_groups	= pci_drv_groups,
1633	.pm		= PCI_PM_OPS_PTR,
1634	.num_vf		= pci_bus_num_vf,
1635	.dma_configure	= pci_dma_configure,
1636};
1637EXPORT_SYMBOL(pci_bus_type);
1638
1639#ifdef CONFIG_PCIEPORTBUS
1640static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
1641{
1642	struct pcie_device *pciedev;
1643	struct pcie_port_service_driver *driver;
1644
1645	if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1646		return 0;
1647
1648	pciedev = to_pcie_device(dev);
1649	driver = to_service_driver(drv);
1650
1651	if (driver->service != pciedev->service)
1652		return 0;
1653
1654	if (driver->port_type != PCIE_ANY_PORT &&
1655	    driver->port_type != pci_pcie_type(pciedev->port))
1656		return 0;
1657
1658	return 1;
1659}
1660
1661struct bus_type pcie_port_bus_type = {
1662	.name		= "pci_express",
1663	.match		= pcie_port_bus_match,
1664};
1665EXPORT_SYMBOL_GPL(pcie_port_bus_type);
1666#endif
1667
1668static int __init pci_driver_init(void)
1669{
1670	int ret;
1671
1672	ret = bus_register(&pci_bus_type);
1673	if (ret)
1674		return ret;
1675
1676#ifdef CONFIG_PCIEPORTBUS
1677	ret = bus_register(&pcie_port_bus_type);
1678	if (ret)
1679		return ret;
1680#endif
1681	dma_debug_add_bus(&pci_bus_type);
1682	return 0;
1683}
1684postcore_initcall(pci_driver_init);