Loading...
1/*
2 * drivers/pci/pci-driver.c
3 *
4 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
5 * (C) Copyright 2007 Novell Inc.
6 *
7 * Released under the GPL v2 only.
8 *
9 */
10
11#include <linux/pci.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/device.h>
15#include <linux/mempolicy.h>
16#include <linux/string.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/cpu.h>
20#include <linux/pm_runtime.h>
21#include <linux/suspend.h>
22#include "pci.h"
23
24struct pci_dynid {
25 struct list_head node;
26 struct pci_device_id id;
27};
28
29/**
30 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
31 * @drv: target pci driver
32 * @vendor: PCI vendor ID
33 * @device: PCI device ID
34 * @subvendor: PCI subvendor ID
35 * @subdevice: PCI subdevice ID
36 * @class: PCI class
37 * @class_mask: PCI class mask
38 * @driver_data: private driver data
39 *
40 * Adds a new dynamic pci device ID to this driver and causes the
41 * driver to probe for all devices again. @drv must have been
42 * registered prior to calling this function.
43 *
44 * CONTEXT:
45 * Does GFP_KERNEL allocation.
46 *
47 * RETURNS:
48 * 0 on success, -errno on failure.
49 */
50int pci_add_dynid(struct pci_driver *drv,
51 unsigned int vendor, unsigned int device,
52 unsigned int subvendor, unsigned int subdevice,
53 unsigned int class, unsigned int class_mask,
54 unsigned long driver_data)
55{
56 struct pci_dynid *dynid;
57 int retval;
58
59 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
60 if (!dynid)
61 return -ENOMEM;
62
63 dynid->id.vendor = vendor;
64 dynid->id.device = device;
65 dynid->id.subvendor = subvendor;
66 dynid->id.subdevice = subdevice;
67 dynid->id.class = class;
68 dynid->id.class_mask = class_mask;
69 dynid->id.driver_data = driver_data;
70
71 spin_lock(&drv->dynids.lock);
72 list_add_tail(&dynid->node, &drv->dynids.list);
73 spin_unlock(&drv->dynids.lock);
74
75 retval = driver_attach(&drv->driver);
76
77 return retval;
78}
79
80static void pci_free_dynids(struct pci_driver *drv)
81{
82 struct pci_dynid *dynid, *n;
83
84 spin_lock(&drv->dynids.lock);
85 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
86 list_del(&dynid->node);
87 kfree(dynid);
88 }
89 spin_unlock(&drv->dynids.lock);
90}
91
92/*
93 * Dynamic device ID manipulation via sysfs is disabled for !CONFIG_HOTPLUG
94 */
95#ifdef CONFIG_HOTPLUG
96/**
97 * store_new_id - sysfs frontend to pci_add_dynid()
98 * @driver: target device driver
99 * @buf: buffer for scanning device ID data
100 * @count: input size
101 *
102 * Allow PCI IDs to be added to an existing driver via sysfs.
103 */
104static ssize_t
105store_new_id(struct device_driver *driver, const char *buf, size_t count)
106{
107 struct pci_driver *pdrv = to_pci_driver(driver);
108 const struct pci_device_id *ids = pdrv->id_table;
109 __u32 vendor, device, subvendor=PCI_ANY_ID,
110 subdevice=PCI_ANY_ID, class=0, class_mask=0;
111 unsigned long driver_data=0;
112 int fields=0;
113 int retval;
114
115 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
116 &vendor, &device, &subvendor, &subdevice,
117 &class, &class_mask, &driver_data);
118 if (fields < 2)
119 return -EINVAL;
120
121 /* Only accept driver_data values that match an existing id_table
122 entry */
123 if (ids) {
124 retval = -EINVAL;
125 while (ids->vendor || ids->subvendor || ids->class_mask) {
126 if (driver_data == ids->driver_data) {
127 retval = 0;
128 break;
129 }
130 ids++;
131 }
132 if (retval) /* No match */
133 return retval;
134 }
135
136 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
137 class, class_mask, driver_data);
138 if (retval)
139 return retval;
140 return count;
141}
142static DRIVER_ATTR(new_id, S_IWUSR, NULL, store_new_id);
143
144/**
145 * store_remove_id - remove a PCI device ID from this driver
146 * @driver: target device driver
147 * @buf: buffer for scanning device ID data
148 * @count: input size
149 *
150 * Removes a dynamic pci device ID to this driver.
151 */
152static ssize_t
153store_remove_id(struct device_driver *driver, const char *buf, size_t count)
154{
155 struct pci_dynid *dynid, *n;
156 struct pci_driver *pdrv = to_pci_driver(driver);
157 __u32 vendor, device, subvendor = PCI_ANY_ID,
158 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
159 int fields = 0;
160 int retval = -ENODEV;
161
162 fields = sscanf(buf, "%x %x %x %x %x %x",
163 &vendor, &device, &subvendor, &subdevice,
164 &class, &class_mask);
165 if (fields < 2)
166 return -EINVAL;
167
168 spin_lock(&pdrv->dynids.lock);
169 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
170 struct pci_device_id *id = &dynid->id;
171 if ((id->vendor == vendor) &&
172 (id->device == device) &&
173 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
174 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
175 !((id->class ^ class) & class_mask)) {
176 list_del(&dynid->node);
177 kfree(dynid);
178 retval = 0;
179 break;
180 }
181 }
182 spin_unlock(&pdrv->dynids.lock);
183
184 if (retval)
185 return retval;
186 return count;
187}
188static DRIVER_ATTR(remove_id, S_IWUSR, NULL, store_remove_id);
189
190static int
191pci_create_newid_files(struct pci_driver *drv)
192{
193 int error = 0;
194
195 if (drv->probe != NULL) {
196 error = driver_create_file(&drv->driver, &driver_attr_new_id);
197 if (error == 0) {
198 error = driver_create_file(&drv->driver,
199 &driver_attr_remove_id);
200 if (error)
201 driver_remove_file(&drv->driver,
202 &driver_attr_new_id);
203 }
204 }
205 return error;
206}
207
208static void pci_remove_newid_files(struct pci_driver *drv)
209{
210 driver_remove_file(&drv->driver, &driver_attr_remove_id);
211 driver_remove_file(&drv->driver, &driver_attr_new_id);
212}
213#else /* !CONFIG_HOTPLUG */
214static inline int pci_create_newid_files(struct pci_driver *drv)
215{
216 return 0;
217}
218static inline void pci_remove_newid_files(struct pci_driver *drv) {}
219#endif
220
221/**
222 * pci_match_id - See if a pci device matches a given pci_id table
223 * @ids: array of PCI device id structures to search in
224 * @dev: the PCI device structure to match against.
225 *
226 * Used by a driver to check whether a PCI device present in the
227 * system is in its list of supported devices. Returns the matching
228 * pci_device_id structure or %NULL if there is no match.
229 *
230 * Deprecated, don't use this as it will not catch any dynamic ids
231 * that a driver might want to check for.
232 */
233const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
234 struct pci_dev *dev)
235{
236 if (ids) {
237 while (ids->vendor || ids->subvendor || ids->class_mask) {
238 if (pci_match_one_device(ids, dev))
239 return ids;
240 ids++;
241 }
242 }
243 return NULL;
244}
245
246/**
247 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
248 * @drv: the PCI driver to match against
249 * @dev: the PCI device structure to match against
250 *
251 * Used by a driver to check whether a PCI device present in the
252 * system is in its list of supported devices. Returns the matching
253 * pci_device_id structure or %NULL if there is no match.
254 */
255static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
256 struct pci_dev *dev)
257{
258 struct pci_dynid *dynid;
259
260 /* Look at the dynamic ids first, before the static ones */
261 spin_lock(&drv->dynids.lock);
262 list_for_each_entry(dynid, &drv->dynids.list, node) {
263 if (pci_match_one_device(&dynid->id, dev)) {
264 spin_unlock(&drv->dynids.lock);
265 return &dynid->id;
266 }
267 }
268 spin_unlock(&drv->dynids.lock);
269
270 return pci_match_id(drv->id_table, dev);
271}
272
273struct drv_dev_and_id {
274 struct pci_driver *drv;
275 struct pci_dev *dev;
276 const struct pci_device_id *id;
277};
278
279static long local_pci_probe(void *_ddi)
280{
281 struct drv_dev_and_id *ddi = _ddi;
282 struct device *dev = &ddi->dev->dev;
283 int rc;
284
285 /* Unbound PCI devices are always set to disabled and suspended.
286 * During probe, the device is set to enabled and active and the
287 * usage count is incremented. If the driver supports runtime PM,
288 * it should call pm_runtime_put_noidle() in its probe routine and
289 * pm_runtime_get_noresume() in its remove routine.
290 */
291 pm_runtime_get_noresume(dev);
292 pm_runtime_set_active(dev);
293 pm_runtime_enable(dev);
294
295 rc = ddi->drv->probe(ddi->dev, ddi->id);
296 if (rc) {
297 pm_runtime_disable(dev);
298 pm_runtime_set_suspended(dev);
299 pm_runtime_put_noidle(dev);
300 }
301 return rc;
302}
303
304static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
305 const struct pci_device_id *id)
306{
307 int error, node;
308 struct drv_dev_and_id ddi = { drv, dev, id };
309
310 /* Execute driver initialization on node where the device's
311 bus is attached to. This way the driver likely allocates
312 its local memory on the right node without any need to
313 change it. */
314 node = dev_to_node(&dev->dev);
315 if (node >= 0) {
316 int cpu;
317
318 get_online_cpus();
319 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
320 if (cpu < nr_cpu_ids)
321 error = work_on_cpu(cpu, local_pci_probe, &ddi);
322 else
323 error = local_pci_probe(&ddi);
324 put_online_cpus();
325 } else
326 error = local_pci_probe(&ddi);
327 return error;
328}
329
330/**
331 * __pci_device_probe - check if a driver wants to claim a specific PCI device
332 * @drv: driver to call to check if it wants the PCI device
333 * @pci_dev: PCI device being probed
334 *
335 * returns 0 on success, else error.
336 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
337 */
338static int
339__pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
340{
341 const struct pci_device_id *id;
342 int error = 0;
343
344 if (!pci_dev->driver && drv->probe) {
345 error = -ENODEV;
346
347 id = pci_match_device(drv, pci_dev);
348 if (id)
349 error = pci_call_probe(drv, pci_dev, id);
350 if (error >= 0) {
351 pci_dev->driver = drv;
352 error = 0;
353 }
354 }
355 return error;
356}
357
358static int pci_device_probe(struct device * dev)
359{
360 int error = 0;
361 struct pci_driver *drv;
362 struct pci_dev *pci_dev;
363
364 drv = to_pci_driver(dev->driver);
365 pci_dev = to_pci_dev(dev);
366 pci_dev_get(pci_dev);
367 error = __pci_device_probe(drv, pci_dev);
368 if (error)
369 pci_dev_put(pci_dev);
370
371 return error;
372}
373
374static int pci_device_remove(struct device * dev)
375{
376 struct pci_dev * pci_dev = to_pci_dev(dev);
377 struct pci_driver * drv = pci_dev->driver;
378
379 if (drv) {
380 if (drv->remove) {
381 pm_runtime_get_sync(dev);
382 drv->remove(pci_dev);
383 pm_runtime_put_noidle(dev);
384 }
385 pci_dev->driver = NULL;
386 }
387
388 /* Undo the runtime PM settings in local_pci_probe() */
389 pm_runtime_disable(dev);
390 pm_runtime_set_suspended(dev);
391 pm_runtime_put_noidle(dev);
392
393 /*
394 * If the device is still on, set the power state as "unknown",
395 * since it might change by the next time we load the driver.
396 */
397 if (pci_dev->current_state == PCI_D0)
398 pci_dev->current_state = PCI_UNKNOWN;
399
400 /*
401 * We would love to complain here if pci_dev->is_enabled is set, that
402 * the driver should have called pci_disable_device(), but the
403 * unfortunate fact is there are too many odd BIOS and bridge setups
404 * that don't like drivers doing that all of the time.
405 * Oh well, we can dream of sane hardware when we sleep, no matter how
406 * horrible the crap we have to deal with is when we are awake...
407 */
408
409 pci_dev_put(pci_dev);
410 return 0;
411}
412
413static void pci_device_shutdown(struct device *dev)
414{
415 struct pci_dev *pci_dev = to_pci_dev(dev);
416 struct pci_driver *drv = pci_dev->driver;
417
418 if (drv && drv->shutdown)
419 drv->shutdown(pci_dev);
420 pci_msi_shutdown(pci_dev);
421 pci_msix_shutdown(pci_dev);
422
423 /*
424 * Turn off Bus Master bit on the device to tell it to not
425 * continue to do DMA
426 */
427 pci_disable_device(pci_dev);
428
429 /*
430 * Devices may be enabled to wake up by runtime PM, but they need not
431 * be supposed to wake up the system from its "power off" state (e.g.
432 * ACPI S5). Therefore disable wakeup for all devices that aren't
433 * supposed to wake up the system at this point. The state argument
434 * will be ignored by pci_enable_wake().
435 */
436 if (!device_may_wakeup(dev))
437 pci_enable_wake(pci_dev, PCI_UNKNOWN, false);
438}
439
440#ifdef CONFIG_PM
441
442/* Auxiliary functions used for system resume and run-time resume. */
443
444/**
445 * pci_restore_standard_config - restore standard config registers of PCI device
446 * @pci_dev: PCI device to handle
447 */
448static int pci_restore_standard_config(struct pci_dev *pci_dev)
449{
450 pci_update_current_state(pci_dev, PCI_UNKNOWN);
451
452 if (pci_dev->current_state != PCI_D0) {
453 int error = pci_set_power_state(pci_dev, PCI_D0);
454 if (error)
455 return error;
456 }
457
458 pci_restore_state(pci_dev);
459 return 0;
460}
461
462static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
463{
464 pci_restore_standard_config(pci_dev);
465 pci_fixup_device(pci_fixup_resume_early, pci_dev);
466}
467
468#endif
469
470#ifdef CONFIG_PM_SLEEP
471
472/*
473 * Default "suspend" method for devices that have no driver provided suspend,
474 * or not even a driver at all (second part).
475 */
476static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
477{
478 /*
479 * mark its power state as "unknown", since we don't know if
480 * e.g. the BIOS will change its device state when we suspend.
481 */
482 if (pci_dev->current_state == PCI_D0)
483 pci_dev->current_state = PCI_UNKNOWN;
484}
485
486/*
487 * Default "resume" method for devices that have no driver provided resume,
488 * or not even a driver at all (second part).
489 */
490static int pci_pm_reenable_device(struct pci_dev *pci_dev)
491{
492 int retval;
493
494 /* if the device was enabled before suspend, reenable */
495 retval = pci_reenable_device(pci_dev);
496 /*
497 * if the device was busmaster before the suspend, make it busmaster
498 * again
499 */
500 if (pci_dev->is_busmaster)
501 pci_set_master(pci_dev);
502
503 return retval;
504}
505
506static int pci_legacy_suspend(struct device *dev, pm_message_t state)
507{
508 struct pci_dev * pci_dev = to_pci_dev(dev);
509 struct pci_driver * drv = pci_dev->driver;
510
511 if (drv && drv->suspend) {
512 pci_power_t prev = pci_dev->current_state;
513 int error;
514
515 error = drv->suspend(pci_dev, state);
516 suspend_report_result(drv->suspend, error);
517 if (error)
518 return error;
519
520 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
521 && pci_dev->current_state != PCI_UNKNOWN) {
522 WARN_ONCE(pci_dev->current_state != prev,
523 "PCI PM: Device state not saved by %pF\n",
524 drv->suspend);
525 }
526 }
527
528 pci_fixup_device(pci_fixup_suspend, pci_dev);
529
530 return 0;
531}
532
533static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
534{
535 struct pci_dev * pci_dev = to_pci_dev(dev);
536 struct pci_driver * drv = pci_dev->driver;
537
538 if (drv && drv->suspend_late) {
539 pci_power_t prev = pci_dev->current_state;
540 int error;
541
542 error = drv->suspend_late(pci_dev, state);
543 suspend_report_result(drv->suspend_late, error);
544 if (error)
545 return error;
546
547 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
548 && pci_dev->current_state != PCI_UNKNOWN) {
549 WARN_ONCE(pci_dev->current_state != prev,
550 "PCI PM: Device state not saved by %pF\n",
551 drv->suspend_late);
552 return 0;
553 }
554 }
555
556 if (!pci_dev->state_saved)
557 pci_save_state(pci_dev);
558
559 pci_pm_set_unknown_state(pci_dev);
560
561 return 0;
562}
563
564static int pci_legacy_resume_early(struct device *dev)
565{
566 struct pci_dev * pci_dev = to_pci_dev(dev);
567 struct pci_driver * drv = pci_dev->driver;
568
569 return drv && drv->resume_early ?
570 drv->resume_early(pci_dev) : 0;
571}
572
573static int pci_legacy_resume(struct device *dev)
574{
575 struct pci_dev * pci_dev = to_pci_dev(dev);
576 struct pci_driver * drv = pci_dev->driver;
577
578 pci_fixup_device(pci_fixup_resume, pci_dev);
579
580 return drv && drv->resume ?
581 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
582}
583
584/* Auxiliary functions used by the new power management framework */
585
586static void pci_pm_default_resume(struct pci_dev *pci_dev)
587{
588 pci_fixup_device(pci_fixup_resume, pci_dev);
589
590 if (!pci_is_bridge(pci_dev))
591 pci_enable_wake(pci_dev, PCI_D0, false);
592}
593
594static void pci_pm_default_suspend(struct pci_dev *pci_dev)
595{
596 /* Disable non-bridge devices without PM support */
597 if (!pci_is_bridge(pci_dev))
598 pci_disable_enabled_device(pci_dev);
599}
600
601static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
602{
603 struct pci_driver *drv = pci_dev->driver;
604 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
605 || drv->resume_early);
606
607 /*
608 * Legacy PM support is used by default, so warn if the new framework is
609 * supported as well. Drivers are supposed to support either the
610 * former, or the latter, but not both at the same time.
611 */
612 WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
613 drv->name, pci_dev->vendor, pci_dev->device);
614
615 return ret;
616}
617
618/* New power management framework */
619
620static int pci_pm_prepare(struct device *dev)
621{
622 struct device_driver *drv = dev->driver;
623 int error = 0;
624
625 /*
626 * If a PCI device configured to wake up the system from sleep states
627 * has been suspended at run time and there's a resume request pending
628 * for it, this is equivalent to the device signaling wakeup, so the
629 * system suspend operation should be aborted.
630 */
631 pm_runtime_get_noresume(dev);
632 if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
633 pm_wakeup_event(dev, 0);
634
635 if (pm_wakeup_pending()) {
636 pm_runtime_put_sync(dev);
637 return -EBUSY;
638 }
639
640 /*
641 * PCI devices suspended at run time need to be resumed at this
642 * point, because in general it is necessary to reconfigure them for
643 * system suspend. Namely, if the device is supposed to wake up the
644 * system from the sleep state, we may need to reconfigure it for this
645 * purpose. In turn, if the device is not supposed to wake up the
646 * system from the sleep state, we'll have to prevent it from signaling
647 * wake-up.
648 */
649 pm_runtime_resume(dev);
650
651 if (drv && drv->pm && drv->pm->prepare)
652 error = drv->pm->prepare(dev);
653
654 return error;
655}
656
657static void pci_pm_complete(struct device *dev)
658{
659 struct device_driver *drv = dev->driver;
660
661 if (drv && drv->pm && drv->pm->complete)
662 drv->pm->complete(dev);
663
664 pm_runtime_put_sync(dev);
665}
666
667#else /* !CONFIG_PM_SLEEP */
668
669#define pci_pm_prepare NULL
670#define pci_pm_complete NULL
671
672#endif /* !CONFIG_PM_SLEEP */
673
674#ifdef CONFIG_SUSPEND
675
676static int pci_pm_suspend(struct device *dev)
677{
678 struct pci_dev *pci_dev = to_pci_dev(dev);
679 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
680
681 if (pci_has_legacy_pm_support(pci_dev))
682 return pci_legacy_suspend(dev, PMSG_SUSPEND);
683
684 if (!pm) {
685 pci_pm_default_suspend(pci_dev);
686 goto Fixup;
687 }
688
689 if (pm->suspend) {
690 pci_power_t prev = pci_dev->current_state;
691 int error;
692
693 error = pm->suspend(dev);
694 suspend_report_result(pm->suspend, error);
695 if (error)
696 return error;
697
698 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
699 && pci_dev->current_state != PCI_UNKNOWN) {
700 WARN_ONCE(pci_dev->current_state != prev,
701 "PCI PM: State of device not saved by %pF\n",
702 pm->suspend);
703 }
704 }
705
706 Fixup:
707 pci_fixup_device(pci_fixup_suspend, pci_dev);
708
709 return 0;
710}
711
712static int pci_pm_suspend_noirq(struct device *dev)
713{
714 struct pci_dev *pci_dev = to_pci_dev(dev);
715 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
716
717 if (pci_has_legacy_pm_support(pci_dev))
718 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
719
720 if (!pm) {
721 pci_save_state(pci_dev);
722 return 0;
723 }
724
725 if (pm->suspend_noirq) {
726 pci_power_t prev = pci_dev->current_state;
727 int error;
728
729 error = pm->suspend_noirq(dev);
730 suspend_report_result(pm->suspend_noirq, error);
731 if (error)
732 return error;
733
734 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
735 && pci_dev->current_state != PCI_UNKNOWN) {
736 WARN_ONCE(pci_dev->current_state != prev,
737 "PCI PM: State of device not saved by %pF\n",
738 pm->suspend_noirq);
739 return 0;
740 }
741 }
742
743 if (!pci_dev->state_saved) {
744 pci_save_state(pci_dev);
745 if (!pci_is_bridge(pci_dev))
746 pci_prepare_to_sleep(pci_dev);
747 }
748
749 pci_pm_set_unknown_state(pci_dev);
750
751 /*
752 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
753 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
754 * hasn't been quiesced and tries to turn it off. If the controller
755 * is already in D3, this can hang or cause memory corruption.
756 *
757 * Since the value of the COMMAND register doesn't matter once the
758 * device has been suspended, we can safely set it to 0 here.
759 */
760 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
761 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
762
763 return 0;
764}
765
766static int pci_pm_resume_noirq(struct device *dev)
767{
768 struct pci_dev *pci_dev = to_pci_dev(dev);
769 struct device_driver *drv = dev->driver;
770 int error = 0;
771
772 pci_pm_default_resume_early(pci_dev);
773
774 if (pci_has_legacy_pm_support(pci_dev))
775 return pci_legacy_resume_early(dev);
776
777 if (drv && drv->pm && drv->pm->resume_noirq)
778 error = drv->pm->resume_noirq(dev);
779
780 return error;
781}
782
783static int pci_pm_resume(struct device *dev)
784{
785 struct pci_dev *pci_dev = to_pci_dev(dev);
786 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
787 int error = 0;
788
789 /*
790 * This is necessary for the suspend error path in which resume is
791 * called without restoring the standard config registers of the device.
792 */
793 if (pci_dev->state_saved)
794 pci_restore_standard_config(pci_dev);
795
796 if (pci_has_legacy_pm_support(pci_dev))
797 return pci_legacy_resume(dev);
798
799 pci_pm_default_resume(pci_dev);
800
801 if (pm) {
802 if (pm->resume)
803 error = pm->resume(dev);
804 } else {
805 pci_pm_reenable_device(pci_dev);
806 }
807
808 return error;
809}
810
811#else /* !CONFIG_SUSPEND */
812
813#define pci_pm_suspend NULL
814#define pci_pm_suspend_noirq NULL
815#define pci_pm_resume NULL
816#define pci_pm_resume_noirq NULL
817
818#endif /* !CONFIG_SUSPEND */
819
820#ifdef CONFIG_HIBERNATE_CALLBACKS
821
822static int pci_pm_freeze(struct device *dev)
823{
824 struct pci_dev *pci_dev = to_pci_dev(dev);
825 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
826
827 if (pci_has_legacy_pm_support(pci_dev))
828 return pci_legacy_suspend(dev, PMSG_FREEZE);
829
830 if (!pm) {
831 pci_pm_default_suspend(pci_dev);
832 return 0;
833 }
834
835 if (pm->freeze) {
836 int error;
837
838 error = pm->freeze(dev);
839 suspend_report_result(pm->freeze, error);
840 if (error)
841 return error;
842 }
843
844 return 0;
845}
846
847static int pci_pm_freeze_noirq(struct device *dev)
848{
849 struct pci_dev *pci_dev = to_pci_dev(dev);
850 struct device_driver *drv = dev->driver;
851
852 if (pci_has_legacy_pm_support(pci_dev))
853 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
854
855 if (drv && drv->pm && drv->pm->freeze_noirq) {
856 int error;
857
858 error = drv->pm->freeze_noirq(dev);
859 suspend_report_result(drv->pm->freeze_noirq, error);
860 if (error)
861 return error;
862 }
863
864 if (!pci_dev->state_saved)
865 pci_save_state(pci_dev);
866
867 pci_pm_set_unknown_state(pci_dev);
868
869 return 0;
870}
871
872static int pci_pm_thaw_noirq(struct device *dev)
873{
874 struct pci_dev *pci_dev = to_pci_dev(dev);
875 struct device_driver *drv = dev->driver;
876 int error = 0;
877
878 if (pci_has_legacy_pm_support(pci_dev))
879 return pci_legacy_resume_early(dev);
880
881 pci_update_current_state(pci_dev, PCI_D0);
882
883 if (drv && drv->pm && drv->pm->thaw_noirq)
884 error = drv->pm->thaw_noirq(dev);
885
886 return error;
887}
888
889static int pci_pm_thaw(struct device *dev)
890{
891 struct pci_dev *pci_dev = to_pci_dev(dev);
892 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
893 int error = 0;
894
895 if (pci_has_legacy_pm_support(pci_dev))
896 return pci_legacy_resume(dev);
897
898 if (pm) {
899 if (pm->thaw)
900 error = pm->thaw(dev);
901 } else {
902 pci_pm_reenable_device(pci_dev);
903 }
904
905 pci_dev->state_saved = false;
906
907 return error;
908}
909
910static int pci_pm_poweroff(struct device *dev)
911{
912 struct pci_dev *pci_dev = to_pci_dev(dev);
913 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
914
915 if (pci_has_legacy_pm_support(pci_dev))
916 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
917
918 if (!pm) {
919 pci_pm_default_suspend(pci_dev);
920 goto Fixup;
921 }
922
923 if (pm->poweroff) {
924 int error;
925
926 error = pm->poweroff(dev);
927 suspend_report_result(pm->poweroff, error);
928 if (error)
929 return error;
930 }
931
932 Fixup:
933 pci_fixup_device(pci_fixup_suspend, pci_dev);
934
935 return 0;
936}
937
938static int pci_pm_poweroff_noirq(struct device *dev)
939{
940 struct pci_dev *pci_dev = to_pci_dev(dev);
941 struct device_driver *drv = dev->driver;
942
943 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
944 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
945
946 if (!drv || !drv->pm)
947 return 0;
948
949 if (drv->pm->poweroff_noirq) {
950 int error;
951
952 error = drv->pm->poweroff_noirq(dev);
953 suspend_report_result(drv->pm->poweroff_noirq, error);
954 if (error)
955 return error;
956 }
957
958 if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
959 pci_prepare_to_sleep(pci_dev);
960
961 /*
962 * The reason for doing this here is the same as for the analogous code
963 * in pci_pm_suspend_noirq().
964 */
965 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
966 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
967
968 return 0;
969}
970
971static int pci_pm_restore_noirq(struct device *dev)
972{
973 struct pci_dev *pci_dev = to_pci_dev(dev);
974 struct device_driver *drv = dev->driver;
975 int error = 0;
976
977 pci_pm_default_resume_early(pci_dev);
978
979 if (pci_has_legacy_pm_support(pci_dev))
980 return pci_legacy_resume_early(dev);
981
982 if (drv && drv->pm && drv->pm->restore_noirq)
983 error = drv->pm->restore_noirq(dev);
984
985 return error;
986}
987
988static int pci_pm_restore(struct device *dev)
989{
990 struct pci_dev *pci_dev = to_pci_dev(dev);
991 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
992 int error = 0;
993
994 /*
995 * This is necessary for the hibernation error path in which restore is
996 * called without restoring the standard config registers of the device.
997 */
998 if (pci_dev->state_saved)
999 pci_restore_standard_config(pci_dev);
1000
1001 if (pci_has_legacy_pm_support(pci_dev))
1002 return pci_legacy_resume(dev);
1003
1004 pci_pm_default_resume(pci_dev);
1005
1006 if (pm) {
1007 if (pm->restore)
1008 error = pm->restore(dev);
1009 } else {
1010 pci_pm_reenable_device(pci_dev);
1011 }
1012
1013 return error;
1014}
1015
1016#else /* !CONFIG_HIBERNATE_CALLBACKS */
1017
1018#define pci_pm_freeze NULL
1019#define pci_pm_freeze_noirq NULL
1020#define pci_pm_thaw NULL
1021#define pci_pm_thaw_noirq NULL
1022#define pci_pm_poweroff NULL
1023#define pci_pm_poweroff_noirq NULL
1024#define pci_pm_restore NULL
1025#define pci_pm_restore_noirq NULL
1026
1027#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1028
1029#ifdef CONFIG_PM_RUNTIME
1030
1031static int pci_pm_runtime_suspend(struct device *dev)
1032{
1033 struct pci_dev *pci_dev = to_pci_dev(dev);
1034 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1035 pci_power_t prev = pci_dev->current_state;
1036 int error;
1037
1038 if (!pm || !pm->runtime_suspend)
1039 return -ENOSYS;
1040
1041 error = pm->runtime_suspend(dev);
1042 suspend_report_result(pm->runtime_suspend, error);
1043 if (error)
1044 return error;
1045
1046 pci_fixup_device(pci_fixup_suspend, pci_dev);
1047
1048 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1049 && pci_dev->current_state != PCI_UNKNOWN) {
1050 WARN_ONCE(pci_dev->current_state != prev,
1051 "PCI PM: State of device not saved by %pF\n",
1052 pm->runtime_suspend);
1053 return 0;
1054 }
1055
1056 if (!pci_dev->state_saved)
1057 pci_save_state(pci_dev);
1058
1059 pci_finish_runtime_suspend(pci_dev);
1060
1061 return 0;
1062}
1063
1064static int pci_pm_runtime_resume(struct device *dev)
1065{
1066 struct pci_dev *pci_dev = to_pci_dev(dev);
1067 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1068
1069 if (!pm || !pm->runtime_resume)
1070 return -ENOSYS;
1071
1072 pci_pm_default_resume_early(pci_dev);
1073 __pci_enable_wake(pci_dev, PCI_D0, true, false);
1074 pci_fixup_device(pci_fixup_resume, pci_dev);
1075
1076 return pm->runtime_resume(dev);
1077}
1078
1079static int pci_pm_runtime_idle(struct device *dev)
1080{
1081 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1082
1083 if (!pm)
1084 return -ENOSYS;
1085
1086 if (pm->runtime_idle) {
1087 int ret = pm->runtime_idle(dev);
1088 if (ret)
1089 return ret;
1090 }
1091
1092 pm_runtime_suspend(dev);
1093
1094 return 0;
1095}
1096
1097#else /* !CONFIG_PM_RUNTIME */
1098
1099#define pci_pm_runtime_suspend NULL
1100#define pci_pm_runtime_resume NULL
1101#define pci_pm_runtime_idle NULL
1102
1103#endif /* !CONFIG_PM_RUNTIME */
1104
1105#ifdef CONFIG_PM
1106
1107const struct dev_pm_ops pci_dev_pm_ops = {
1108 .prepare = pci_pm_prepare,
1109 .complete = pci_pm_complete,
1110 .suspend = pci_pm_suspend,
1111 .resume = pci_pm_resume,
1112 .freeze = pci_pm_freeze,
1113 .thaw = pci_pm_thaw,
1114 .poweroff = pci_pm_poweroff,
1115 .restore = pci_pm_restore,
1116 .suspend_noirq = pci_pm_suspend_noirq,
1117 .resume_noirq = pci_pm_resume_noirq,
1118 .freeze_noirq = pci_pm_freeze_noirq,
1119 .thaw_noirq = pci_pm_thaw_noirq,
1120 .poweroff_noirq = pci_pm_poweroff_noirq,
1121 .restore_noirq = pci_pm_restore_noirq,
1122 .runtime_suspend = pci_pm_runtime_suspend,
1123 .runtime_resume = pci_pm_runtime_resume,
1124 .runtime_idle = pci_pm_runtime_idle,
1125};
1126
1127#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1128
1129#else /* !COMFIG_PM_OPS */
1130
1131#define PCI_PM_OPS_PTR NULL
1132
1133#endif /* !COMFIG_PM_OPS */
1134
1135/**
1136 * __pci_register_driver - register a new pci driver
1137 * @drv: the driver structure to register
1138 * @owner: owner module of drv
1139 * @mod_name: module name string
1140 *
1141 * Adds the driver structure to the list of registered drivers.
1142 * Returns a negative value on error, otherwise 0.
1143 * If no error occurred, the driver remains registered even if
1144 * no device was claimed during registration.
1145 */
1146int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1147 const char *mod_name)
1148{
1149 int error;
1150
1151 /* initialize common driver fields */
1152 drv->driver.name = drv->name;
1153 drv->driver.bus = &pci_bus_type;
1154 drv->driver.owner = owner;
1155 drv->driver.mod_name = mod_name;
1156
1157 spin_lock_init(&drv->dynids.lock);
1158 INIT_LIST_HEAD(&drv->dynids.list);
1159
1160 /* register with core */
1161 error = driver_register(&drv->driver);
1162 if (error)
1163 goto out;
1164
1165 error = pci_create_newid_files(drv);
1166 if (error)
1167 goto out_newid;
1168out:
1169 return error;
1170
1171out_newid:
1172 driver_unregister(&drv->driver);
1173 goto out;
1174}
1175
1176/**
1177 * pci_unregister_driver - unregister a pci driver
1178 * @drv: the driver structure to unregister
1179 *
1180 * Deletes the driver structure from the list of registered PCI drivers,
1181 * gives it a chance to clean up by calling its remove() function for
1182 * each device it was responsible for, and marks those devices as
1183 * driverless.
1184 */
1185
1186void
1187pci_unregister_driver(struct pci_driver *drv)
1188{
1189 pci_remove_newid_files(drv);
1190 driver_unregister(&drv->driver);
1191 pci_free_dynids(drv);
1192}
1193
1194static struct pci_driver pci_compat_driver = {
1195 .name = "compat"
1196};
1197
1198/**
1199 * pci_dev_driver - get the pci_driver of a device
1200 * @dev: the device to query
1201 *
1202 * Returns the appropriate pci_driver structure or %NULL if there is no
1203 * registered driver for the device.
1204 */
1205struct pci_driver *
1206pci_dev_driver(const struct pci_dev *dev)
1207{
1208 if (dev->driver)
1209 return dev->driver;
1210 else {
1211 int i;
1212 for(i=0; i<=PCI_ROM_RESOURCE; i++)
1213 if (dev->resource[i].flags & IORESOURCE_BUSY)
1214 return &pci_compat_driver;
1215 }
1216 return NULL;
1217}
1218
1219/**
1220 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1221 * @dev: the PCI device structure to match against
1222 * @drv: the device driver to search for matching PCI device id structures
1223 *
1224 * Used by a driver to check whether a PCI device present in the
1225 * system is in its list of supported devices. Returns the matching
1226 * pci_device_id structure or %NULL if there is no match.
1227 */
1228static int pci_bus_match(struct device *dev, struct device_driver *drv)
1229{
1230 struct pci_dev *pci_dev = to_pci_dev(dev);
1231 struct pci_driver *pci_drv = to_pci_driver(drv);
1232 const struct pci_device_id *found_id;
1233
1234 found_id = pci_match_device(pci_drv, pci_dev);
1235 if (found_id)
1236 return 1;
1237
1238 return 0;
1239}
1240
1241/**
1242 * pci_dev_get - increments the reference count of the pci device structure
1243 * @dev: the device being referenced
1244 *
1245 * Each live reference to a device should be refcounted.
1246 *
1247 * Drivers for PCI devices should normally record such references in
1248 * their probe() methods, when they bind to a device, and release
1249 * them by calling pci_dev_put(), in their disconnect() methods.
1250 *
1251 * A pointer to the device with the incremented reference counter is returned.
1252 */
1253struct pci_dev *pci_dev_get(struct pci_dev *dev)
1254{
1255 if (dev)
1256 get_device(&dev->dev);
1257 return dev;
1258}
1259
1260/**
1261 * pci_dev_put - release a use of the pci device structure
1262 * @dev: device that's been disconnected
1263 *
1264 * Must be called when a user of a device is finished with it. When the last
1265 * user of the device calls this function, the memory of the device is freed.
1266 */
1267void pci_dev_put(struct pci_dev *dev)
1268{
1269 if (dev)
1270 put_device(&dev->dev);
1271}
1272
1273#ifndef CONFIG_HOTPLUG
1274int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1275{
1276 return -ENODEV;
1277}
1278#endif
1279
1280struct bus_type pci_bus_type = {
1281 .name = "pci",
1282 .match = pci_bus_match,
1283 .uevent = pci_uevent,
1284 .probe = pci_device_probe,
1285 .remove = pci_device_remove,
1286 .shutdown = pci_device_shutdown,
1287 .dev_attrs = pci_dev_attrs,
1288 .bus_attrs = pci_bus_attrs,
1289 .pm = PCI_PM_OPS_PTR,
1290};
1291
1292static int __init pci_driver_init(void)
1293{
1294 return bus_register(&pci_bus_type);
1295}
1296
1297postcore_initcall(pci_driver_init);
1298
1299EXPORT_SYMBOL_GPL(pci_add_dynid);
1300EXPORT_SYMBOL(pci_match_id);
1301EXPORT_SYMBOL(__pci_register_driver);
1302EXPORT_SYMBOL(pci_unregister_driver);
1303EXPORT_SYMBOL(pci_dev_driver);
1304EXPORT_SYMBOL(pci_bus_type);
1305EXPORT_SYMBOL(pci_dev_get);
1306EXPORT_SYMBOL(pci_dev_put);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
4 * (C) Copyright 2007 Novell Inc.
5 */
6
7#include <linux/pci.h>
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/device.h>
11#include <linux/mempolicy.h>
12#include <linux/string.h>
13#include <linux/slab.h>
14#include <linux/sched.h>
15#include <linux/sched/isolation.h>
16#include <linux/cpu.h>
17#include <linux/pm_runtime.h>
18#include <linux/suspend.h>
19#include <linux/kexec.h>
20#include <linux/of_device.h>
21#include <linux/acpi.h>
22#include <linux/dma-map-ops.h>
23#include "pci.h"
24#include "pcie/portdrv.h"
25
26struct pci_dynid {
27 struct list_head node;
28 struct pci_device_id id;
29};
30
31/**
32 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
33 * @drv: target pci driver
34 * @vendor: PCI vendor ID
35 * @device: PCI device ID
36 * @subvendor: PCI subvendor ID
37 * @subdevice: PCI subdevice ID
38 * @class: PCI class
39 * @class_mask: PCI class mask
40 * @driver_data: private driver data
41 *
42 * Adds a new dynamic pci device ID to this driver and causes the
43 * driver to probe for all devices again. @drv must have been
44 * registered prior to calling this function.
45 *
46 * CONTEXT:
47 * Does GFP_KERNEL allocation.
48 *
49 * RETURNS:
50 * 0 on success, -errno on failure.
51 */
52int pci_add_dynid(struct pci_driver *drv,
53 unsigned int vendor, unsigned int device,
54 unsigned int subvendor, unsigned int subdevice,
55 unsigned int class, unsigned int class_mask,
56 unsigned long driver_data)
57{
58 struct pci_dynid *dynid;
59
60 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
61 if (!dynid)
62 return -ENOMEM;
63
64 dynid->id.vendor = vendor;
65 dynid->id.device = device;
66 dynid->id.subvendor = subvendor;
67 dynid->id.subdevice = subdevice;
68 dynid->id.class = class;
69 dynid->id.class_mask = class_mask;
70 dynid->id.driver_data = driver_data;
71
72 spin_lock(&drv->dynids.lock);
73 list_add_tail(&dynid->node, &drv->dynids.list);
74 spin_unlock(&drv->dynids.lock);
75
76 return driver_attach(&drv->driver);
77}
78EXPORT_SYMBOL_GPL(pci_add_dynid);
79
80static void pci_free_dynids(struct pci_driver *drv)
81{
82 struct pci_dynid *dynid, *n;
83
84 spin_lock(&drv->dynids.lock);
85 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
86 list_del(&dynid->node);
87 kfree(dynid);
88 }
89 spin_unlock(&drv->dynids.lock);
90}
91
92/**
93 * pci_match_id - See if a PCI device matches a given pci_id table
94 * @ids: array of PCI device ID structures to search in
95 * @dev: the PCI device structure to match against.
96 *
97 * Used by a driver to check whether a PCI device is in its list of
98 * supported devices. Returns the matching pci_device_id structure or
99 * %NULL if there is no match.
100 *
101 * Deprecated; don't use this as it will not catch any dynamic IDs
102 * that a driver might want to check for.
103 */
104const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
105 struct pci_dev *dev)
106{
107 if (ids) {
108 while (ids->vendor || ids->subvendor || ids->class_mask) {
109 if (pci_match_one_device(ids, dev))
110 return ids;
111 ids++;
112 }
113 }
114 return NULL;
115}
116EXPORT_SYMBOL(pci_match_id);
117
118static const struct pci_device_id pci_device_id_any = {
119 .vendor = PCI_ANY_ID,
120 .device = PCI_ANY_ID,
121 .subvendor = PCI_ANY_ID,
122 .subdevice = PCI_ANY_ID,
123};
124
125/**
126 * pci_match_device - See if a device matches a driver's list of IDs
127 * @drv: the PCI driver to match against
128 * @dev: the PCI device structure to match against
129 *
130 * Used by a driver to check whether a PCI device is in its list of
131 * supported devices or in the dynids list, which may have been augmented
132 * via the sysfs "new_id" file. Returns the matching pci_device_id
133 * structure or %NULL if there is no match.
134 */
135static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
136 struct pci_dev *dev)
137{
138 struct pci_dynid *dynid;
139 const struct pci_device_id *found_id = NULL;
140
141 /* When driver_override is set, only bind to the matching driver */
142 if (dev->driver_override && strcmp(dev->driver_override, drv->name))
143 return NULL;
144
145 /* Look at the dynamic ids first, before the static ones */
146 spin_lock(&drv->dynids.lock);
147 list_for_each_entry(dynid, &drv->dynids.list, node) {
148 if (pci_match_one_device(&dynid->id, dev)) {
149 found_id = &dynid->id;
150 break;
151 }
152 }
153 spin_unlock(&drv->dynids.lock);
154
155 if (!found_id)
156 found_id = pci_match_id(drv->id_table, dev);
157
158 /* driver_override will always match, send a dummy id */
159 if (!found_id && dev->driver_override)
160 found_id = &pci_device_id_any;
161
162 return found_id;
163}
164
165/**
166 * new_id_store - sysfs frontend to pci_add_dynid()
167 * @driver: target device driver
168 * @buf: buffer for scanning device ID data
169 * @count: input size
170 *
171 * Allow PCI IDs to be added to an existing driver via sysfs.
172 */
173static ssize_t new_id_store(struct device_driver *driver, const char *buf,
174 size_t count)
175{
176 struct pci_driver *pdrv = to_pci_driver(driver);
177 const struct pci_device_id *ids = pdrv->id_table;
178 u32 vendor, device, subvendor = PCI_ANY_ID,
179 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
180 unsigned long driver_data = 0;
181 int fields = 0;
182 int retval = 0;
183
184 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
185 &vendor, &device, &subvendor, &subdevice,
186 &class, &class_mask, &driver_data);
187 if (fields < 2)
188 return -EINVAL;
189
190 if (fields != 7) {
191 struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
192 if (!pdev)
193 return -ENOMEM;
194
195 pdev->vendor = vendor;
196 pdev->device = device;
197 pdev->subsystem_vendor = subvendor;
198 pdev->subsystem_device = subdevice;
199 pdev->class = class;
200
201 if (pci_match_device(pdrv, pdev))
202 retval = -EEXIST;
203
204 kfree(pdev);
205
206 if (retval)
207 return retval;
208 }
209
210 /* Only accept driver_data values that match an existing id_table
211 entry */
212 if (ids) {
213 retval = -EINVAL;
214 while (ids->vendor || ids->subvendor || ids->class_mask) {
215 if (driver_data == ids->driver_data) {
216 retval = 0;
217 break;
218 }
219 ids++;
220 }
221 if (retval) /* No match */
222 return retval;
223 }
224
225 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
226 class, class_mask, driver_data);
227 if (retval)
228 return retval;
229 return count;
230}
231static DRIVER_ATTR_WO(new_id);
232
233/**
234 * remove_id_store - remove a PCI device ID from this driver
235 * @driver: target device driver
236 * @buf: buffer for scanning device ID data
237 * @count: input size
238 *
239 * Removes a dynamic pci device ID to this driver.
240 */
241static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
242 size_t count)
243{
244 struct pci_dynid *dynid, *n;
245 struct pci_driver *pdrv = to_pci_driver(driver);
246 u32 vendor, device, subvendor = PCI_ANY_ID,
247 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
248 int fields = 0;
249 size_t retval = -ENODEV;
250
251 fields = sscanf(buf, "%x %x %x %x %x %x",
252 &vendor, &device, &subvendor, &subdevice,
253 &class, &class_mask);
254 if (fields < 2)
255 return -EINVAL;
256
257 spin_lock(&pdrv->dynids.lock);
258 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
259 struct pci_device_id *id = &dynid->id;
260 if ((id->vendor == vendor) &&
261 (id->device == device) &&
262 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
263 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
264 !((id->class ^ class) & class_mask)) {
265 list_del(&dynid->node);
266 kfree(dynid);
267 retval = count;
268 break;
269 }
270 }
271 spin_unlock(&pdrv->dynids.lock);
272
273 return retval;
274}
275static DRIVER_ATTR_WO(remove_id);
276
277static struct attribute *pci_drv_attrs[] = {
278 &driver_attr_new_id.attr,
279 &driver_attr_remove_id.attr,
280 NULL,
281};
282ATTRIBUTE_GROUPS(pci_drv);
283
284struct drv_dev_and_id {
285 struct pci_driver *drv;
286 struct pci_dev *dev;
287 const struct pci_device_id *id;
288};
289
290static long local_pci_probe(void *_ddi)
291{
292 struct drv_dev_and_id *ddi = _ddi;
293 struct pci_dev *pci_dev = ddi->dev;
294 struct pci_driver *pci_drv = ddi->drv;
295 struct device *dev = &pci_dev->dev;
296 int rc;
297
298 /*
299 * Unbound PCI devices are always put in D0, regardless of
300 * runtime PM status. During probe, the device is set to
301 * active and the usage count is incremented. If the driver
302 * supports runtime PM, it should call pm_runtime_put_noidle(),
303 * or any other runtime PM helper function decrementing the usage
304 * count, in its probe routine and pm_runtime_get_noresume() in
305 * its remove routine.
306 */
307 pm_runtime_get_sync(dev);
308 pci_dev->driver = pci_drv;
309 rc = pci_drv->probe(pci_dev, ddi->id);
310 if (!rc)
311 return rc;
312 if (rc < 0) {
313 pci_dev->driver = NULL;
314 pm_runtime_put_sync(dev);
315 return rc;
316 }
317 /*
318 * Probe function should return < 0 for failure, 0 for success
319 * Treat values > 0 as success, but warn.
320 */
321 pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n",
322 rc);
323 return 0;
324}
325
326static bool pci_physfn_is_probed(struct pci_dev *dev)
327{
328#ifdef CONFIG_PCI_IOV
329 return dev->is_virtfn && dev->physfn->is_probed;
330#else
331 return false;
332#endif
333}
334
335static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
336 const struct pci_device_id *id)
337{
338 int error, node, cpu;
339 int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
340 struct drv_dev_and_id ddi = { drv, dev, id };
341
342 /*
343 * Execute driver initialization on node where the device is
344 * attached. This way the driver likely allocates its local memory
345 * on the right node.
346 */
347 node = dev_to_node(&dev->dev);
348 dev->is_probed = 1;
349
350 cpu_hotplug_disable();
351
352 /*
353 * Prevent nesting work_on_cpu() for the case where a Virtual Function
354 * device is probed from work_on_cpu() of the Physical device.
355 */
356 if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
357 pci_physfn_is_probed(dev))
358 cpu = nr_cpu_ids;
359 else
360 cpu = cpumask_any_and(cpumask_of_node(node),
361 housekeeping_cpumask(hk_flags));
362
363 if (cpu < nr_cpu_ids)
364 error = work_on_cpu(cpu, local_pci_probe, &ddi);
365 else
366 error = local_pci_probe(&ddi);
367
368 dev->is_probed = 0;
369 cpu_hotplug_enable();
370 return error;
371}
372
373/**
374 * __pci_device_probe - check if a driver wants to claim a specific PCI device
375 * @drv: driver to call to check if it wants the PCI device
376 * @pci_dev: PCI device being probed
377 *
378 * returns 0 on success, else error.
379 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
380 */
381static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
382{
383 const struct pci_device_id *id;
384 int error = 0;
385
386 if (!pci_dev->driver && drv->probe) {
387 error = -ENODEV;
388
389 id = pci_match_device(drv, pci_dev);
390 if (id)
391 error = pci_call_probe(drv, pci_dev, id);
392 }
393 return error;
394}
395
396int __weak pcibios_alloc_irq(struct pci_dev *dev)
397{
398 return 0;
399}
400
401void __weak pcibios_free_irq(struct pci_dev *dev)
402{
403}
404
405#ifdef CONFIG_PCI_IOV
406static inline bool pci_device_can_probe(struct pci_dev *pdev)
407{
408 return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe ||
409 pdev->driver_override);
410}
411#else
412static inline bool pci_device_can_probe(struct pci_dev *pdev)
413{
414 return true;
415}
416#endif
417
418static int pci_device_probe(struct device *dev)
419{
420 int error;
421 struct pci_dev *pci_dev = to_pci_dev(dev);
422 struct pci_driver *drv = to_pci_driver(dev->driver);
423
424 if (!pci_device_can_probe(pci_dev))
425 return -ENODEV;
426
427 pci_assign_irq(pci_dev);
428
429 error = pcibios_alloc_irq(pci_dev);
430 if (error < 0)
431 return error;
432
433 pci_dev_get(pci_dev);
434 error = __pci_device_probe(drv, pci_dev);
435 if (error) {
436 pcibios_free_irq(pci_dev);
437 pci_dev_put(pci_dev);
438 }
439
440 return error;
441}
442
443static int pci_device_remove(struct device *dev)
444{
445 struct pci_dev *pci_dev = to_pci_dev(dev);
446 struct pci_driver *drv = pci_dev->driver;
447
448 if (drv) {
449 if (drv->remove) {
450 pm_runtime_get_sync(dev);
451 drv->remove(pci_dev);
452 pm_runtime_put_noidle(dev);
453 }
454 pcibios_free_irq(pci_dev);
455 pci_dev->driver = NULL;
456 pci_iov_remove(pci_dev);
457 }
458
459 /* Undo the runtime PM settings in local_pci_probe() */
460 pm_runtime_put_sync(dev);
461
462 /*
463 * If the device is still on, set the power state as "unknown",
464 * since it might change by the next time we load the driver.
465 */
466 if (pci_dev->current_state == PCI_D0)
467 pci_dev->current_state = PCI_UNKNOWN;
468
469 /*
470 * We would love to complain here if pci_dev->is_enabled is set, that
471 * the driver should have called pci_disable_device(), but the
472 * unfortunate fact is there are too many odd BIOS and bridge setups
473 * that don't like drivers doing that all of the time.
474 * Oh well, we can dream of sane hardware when we sleep, no matter how
475 * horrible the crap we have to deal with is when we are awake...
476 */
477
478 pci_dev_put(pci_dev);
479 return 0;
480}
481
482static void pci_device_shutdown(struct device *dev)
483{
484 struct pci_dev *pci_dev = to_pci_dev(dev);
485 struct pci_driver *drv = pci_dev->driver;
486
487 pm_runtime_resume(dev);
488
489 if (drv && drv->shutdown)
490 drv->shutdown(pci_dev);
491
492 /*
493 * If this is a kexec reboot, turn off Bus Master bit on the
494 * device to tell it to not continue to do DMA. Don't touch
495 * devices in D3cold or unknown states.
496 * If it is not a kexec reboot, firmware will hit the PCI
497 * devices with big hammer and stop their DMA any way.
498 */
499 if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
500 pci_clear_master(pci_dev);
501}
502
503#ifdef CONFIG_PM
504
505/* Auxiliary functions used for system resume and run-time resume. */
506
507/**
508 * pci_restore_standard_config - restore standard config registers of PCI device
509 * @pci_dev: PCI device to handle
510 */
511static int pci_restore_standard_config(struct pci_dev *pci_dev)
512{
513 pci_update_current_state(pci_dev, PCI_UNKNOWN);
514
515 if (pci_dev->current_state != PCI_D0) {
516 int error = pci_set_power_state(pci_dev, PCI_D0);
517 if (error)
518 return error;
519 }
520
521 pci_restore_state(pci_dev);
522 pci_pme_restore(pci_dev);
523 return 0;
524}
525
526static void pci_pm_default_resume(struct pci_dev *pci_dev)
527{
528 pci_fixup_device(pci_fixup_resume, pci_dev);
529 pci_enable_wake(pci_dev, PCI_D0, false);
530}
531
532#endif
533
534#ifdef CONFIG_PM_SLEEP
535
536static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
537{
538 pci_power_up(pci_dev);
539 pci_update_current_state(pci_dev, PCI_D0);
540 pci_restore_state(pci_dev);
541 pci_pme_restore(pci_dev);
542}
543
544/*
545 * Default "suspend" method for devices that have no driver provided suspend,
546 * or not even a driver at all (second part).
547 */
548static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
549{
550 /*
551 * mark its power state as "unknown", since we don't know if
552 * e.g. the BIOS will change its device state when we suspend.
553 */
554 if (pci_dev->current_state == PCI_D0)
555 pci_dev->current_state = PCI_UNKNOWN;
556}
557
558/*
559 * Default "resume" method for devices that have no driver provided resume,
560 * or not even a driver at all (second part).
561 */
562static int pci_pm_reenable_device(struct pci_dev *pci_dev)
563{
564 int retval;
565
566 /* if the device was enabled before suspend, reenable */
567 retval = pci_reenable_device(pci_dev);
568 /*
569 * if the device was busmaster before the suspend, make it busmaster
570 * again
571 */
572 if (pci_dev->is_busmaster)
573 pci_set_master(pci_dev);
574
575 return retval;
576}
577
578static int pci_legacy_suspend(struct device *dev, pm_message_t state)
579{
580 struct pci_dev *pci_dev = to_pci_dev(dev);
581 struct pci_driver *drv = pci_dev->driver;
582
583 if (drv && drv->suspend) {
584 pci_power_t prev = pci_dev->current_state;
585 int error;
586
587 error = drv->suspend(pci_dev, state);
588 suspend_report_result(drv->suspend, error);
589 if (error)
590 return error;
591
592 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
593 && pci_dev->current_state != PCI_UNKNOWN) {
594 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
595 "PCI PM: Device state not saved by %pS\n",
596 drv->suspend);
597 }
598 }
599
600 pci_fixup_device(pci_fixup_suspend, pci_dev);
601
602 return 0;
603}
604
605static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
606{
607 struct pci_dev *pci_dev = to_pci_dev(dev);
608
609 if (!pci_dev->state_saved)
610 pci_save_state(pci_dev);
611
612 pci_pm_set_unknown_state(pci_dev);
613
614 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
615
616 return 0;
617}
618
619static int pci_legacy_resume(struct device *dev)
620{
621 struct pci_dev *pci_dev = to_pci_dev(dev);
622 struct pci_driver *drv = pci_dev->driver;
623
624 pci_fixup_device(pci_fixup_resume, pci_dev);
625
626 return drv && drv->resume ?
627 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
628}
629
630/* Auxiliary functions used by the new power management framework */
631
632static void pci_pm_default_suspend(struct pci_dev *pci_dev)
633{
634 /* Disable non-bridge devices without PM support */
635 if (!pci_has_subordinate(pci_dev))
636 pci_disable_enabled_device(pci_dev);
637}
638
639static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
640{
641 struct pci_driver *drv = pci_dev->driver;
642 bool ret = drv && (drv->suspend || drv->resume);
643
644 /*
645 * Legacy PM support is used by default, so warn if the new framework is
646 * supported as well. Drivers are supposed to support either the
647 * former, or the latter, but not both at the same time.
648 */
649 pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n",
650 pci_dev->vendor, pci_dev->device);
651
652 return ret;
653}
654
655/* New power management framework */
656
657static int pci_pm_prepare(struct device *dev)
658{
659 struct pci_dev *pci_dev = to_pci_dev(dev);
660 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
661
662 if (pm && pm->prepare) {
663 int error = pm->prepare(dev);
664 if (error < 0)
665 return error;
666
667 if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
668 return 0;
669 }
670 if (pci_dev_need_resume(pci_dev))
671 return 0;
672
673 /*
674 * The PME setting needs to be adjusted here in case the direct-complete
675 * optimization is used with respect to this device.
676 */
677 pci_dev_adjust_pme(pci_dev);
678 return 1;
679}
680
681static void pci_pm_complete(struct device *dev)
682{
683 struct pci_dev *pci_dev = to_pci_dev(dev);
684
685 pci_dev_complete_resume(pci_dev);
686 pm_generic_complete(dev);
687
688 /* Resume device if platform firmware has put it in reset-power-on */
689 if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
690 pci_power_t pre_sleep_state = pci_dev->current_state;
691
692 pci_refresh_power_state(pci_dev);
693 /*
694 * On platforms with ACPI this check may also trigger for
695 * devices sharing power resources if one of those power
696 * resources has been activated as a result of a change of the
697 * power state of another device sharing it. However, in that
698 * case it is also better to resume the device, in general.
699 */
700 if (pci_dev->current_state < pre_sleep_state)
701 pm_request_resume(dev);
702 }
703}
704
705#else /* !CONFIG_PM_SLEEP */
706
707#define pci_pm_prepare NULL
708#define pci_pm_complete NULL
709
710#endif /* !CONFIG_PM_SLEEP */
711
712#ifdef CONFIG_SUSPEND
713static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
714{
715 /*
716 * Some BIOSes forget to clear Root PME Status bits after system
717 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
718 * Clear those bits now just in case (shouldn't hurt).
719 */
720 if (pci_is_pcie(pci_dev) &&
721 (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
722 pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
723 pcie_clear_root_pme_status(pci_dev);
724}
725
726static int pci_pm_suspend(struct device *dev)
727{
728 struct pci_dev *pci_dev = to_pci_dev(dev);
729 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
730
731 pci_dev->skip_bus_pm = false;
732
733 if (pci_has_legacy_pm_support(pci_dev))
734 return pci_legacy_suspend(dev, PMSG_SUSPEND);
735
736 if (!pm) {
737 pci_pm_default_suspend(pci_dev);
738 return 0;
739 }
740
741 /*
742 * PCI devices suspended at run time may need to be resumed at this
743 * point, because in general it may be necessary to reconfigure them for
744 * system suspend. Namely, if the device is expected to wake up the
745 * system from the sleep state, it may have to be reconfigured for this
746 * purpose, or if the device is not expected to wake up the system from
747 * the sleep state, it should be prevented from signaling wakeup events
748 * going forward.
749 *
750 * Also if the driver of the device does not indicate that its system
751 * suspend callbacks can cope with runtime-suspended devices, it is
752 * better to resume the device from runtime suspend here.
753 */
754 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
755 pci_dev_need_resume(pci_dev)) {
756 pm_runtime_resume(dev);
757 pci_dev->state_saved = false;
758 } else {
759 pci_dev_adjust_pme(pci_dev);
760 }
761
762 if (pm->suspend) {
763 pci_power_t prev = pci_dev->current_state;
764 int error;
765
766 error = pm->suspend(dev);
767 suspend_report_result(pm->suspend, error);
768 if (error)
769 return error;
770
771 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
772 && pci_dev->current_state != PCI_UNKNOWN) {
773 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
774 "PCI PM: State of device not saved by %pS\n",
775 pm->suspend);
776 }
777 }
778
779 return 0;
780}
781
782static int pci_pm_suspend_late(struct device *dev)
783{
784 if (dev_pm_skip_suspend(dev))
785 return 0;
786
787 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
788
789 return pm_generic_suspend_late(dev);
790}
791
792static int pci_pm_suspend_noirq(struct device *dev)
793{
794 struct pci_dev *pci_dev = to_pci_dev(dev);
795 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
796
797 if (dev_pm_skip_suspend(dev))
798 return 0;
799
800 if (pci_has_legacy_pm_support(pci_dev))
801 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
802
803 if (!pm) {
804 pci_save_state(pci_dev);
805 goto Fixup;
806 }
807
808 if (pm->suspend_noirq) {
809 pci_power_t prev = pci_dev->current_state;
810 int error;
811
812 error = pm->suspend_noirq(dev);
813 suspend_report_result(pm->suspend_noirq, error);
814 if (error)
815 return error;
816
817 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
818 && pci_dev->current_state != PCI_UNKNOWN) {
819 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
820 "PCI PM: State of device not saved by %pS\n",
821 pm->suspend_noirq);
822 goto Fixup;
823 }
824 }
825
826 if (pci_dev->skip_bus_pm) {
827 /*
828 * Either the device is a bridge with a child in D0 below it, or
829 * the function is running for the second time in a row without
830 * going through full resume, which is possible only during
831 * suspend-to-idle in a spurious wakeup case. The device should
832 * be in D0 at this point, but if it is a bridge, it may be
833 * necessary to save its state.
834 */
835 if (!pci_dev->state_saved)
836 pci_save_state(pci_dev);
837 } else if (!pci_dev->state_saved) {
838 pci_save_state(pci_dev);
839 if (pci_power_manageable(pci_dev))
840 pci_prepare_to_sleep(pci_dev);
841 }
842
843 pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n",
844 pci_power_name(pci_dev->current_state));
845
846 if (pci_dev->current_state == PCI_D0) {
847 pci_dev->skip_bus_pm = true;
848 /*
849 * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any
850 * downstream device is in D0, so avoid changing the power state
851 * of the parent bridge by setting the skip_bus_pm flag for it.
852 */
853 if (pci_dev->bus->self)
854 pci_dev->bus->self->skip_bus_pm = true;
855 }
856
857 if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) {
858 pci_dbg(pci_dev, "PCI PM: Skipped\n");
859 goto Fixup;
860 }
861
862 pci_pm_set_unknown_state(pci_dev);
863
864 /*
865 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
866 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
867 * hasn't been quiesced and tries to turn it off. If the controller
868 * is already in D3, this can hang or cause memory corruption.
869 *
870 * Since the value of the COMMAND register doesn't matter once the
871 * device has been suspended, we can safely set it to 0 here.
872 */
873 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
874 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
875
876Fixup:
877 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
878
879 /*
880 * If the target system sleep state is suspend-to-idle, it is sufficient
881 * to check whether or not the device's wakeup settings are good for
882 * runtime PM. Otherwise, the pm_resume_via_firmware() check will cause
883 * pci_pm_complete() to take care of fixing up the device's state
884 * anyway, if need be.
885 */
886 if (device_can_wakeup(dev) && !device_may_wakeup(dev))
887 dev->power.may_skip_resume = false;
888
889 return 0;
890}
891
892static int pci_pm_resume_noirq(struct device *dev)
893{
894 struct pci_dev *pci_dev = to_pci_dev(dev);
895 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
896 pci_power_t prev_state = pci_dev->current_state;
897 bool skip_bus_pm = pci_dev->skip_bus_pm;
898
899 if (dev_pm_skip_resume(dev))
900 return 0;
901
902 /*
903 * In the suspend-to-idle case, devices left in D0 during suspend will
904 * stay in D0, so it is not necessary to restore or update their
905 * configuration here and attempting to put them into D0 again is
906 * pointless, so avoid doing that.
907 */
908 if (!(skip_bus_pm && pm_suspend_no_platform()))
909 pci_pm_default_resume_early(pci_dev);
910
911 pci_fixup_device(pci_fixup_resume_early, pci_dev);
912 pcie_pme_root_status_cleanup(pci_dev);
913
914 if (!skip_bus_pm && prev_state == PCI_D3cold)
915 pci_bridge_wait_for_secondary_bus(pci_dev);
916
917 if (pci_has_legacy_pm_support(pci_dev))
918 return 0;
919
920 if (pm && pm->resume_noirq)
921 return pm->resume_noirq(dev);
922
923 return 0;
924}
925
926static int pci_pm_resume_early(struct device *dev)
927{
928 if (dev_pm_skip_resume(dev))
929 return 0;
930
931 return pm_generic_resume_early(dev);
932}
933
934static int pci_pm_resume(struct device *dev)
935{
936 struct pci_dev *pci_dev = to_pci_dev(dev);
937 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
938
939 /*
940 * This is necessary for the suspend error path in which resume is
941 * called without restoring the standard config registers of the device.
942 */
943 if (pci_dev->state_saved)
944 pci_restore_standard_config(pci_dev);
945
946 if (pci_has_legacy_pm_support(pci_dev))
947 return pci_legacy_resume(dev);
948
949 pci_pm_default_resume(pci_dev);
950
951 if (pm) {
952 if (pm->resume)
953 return pm->resume(dev);
954 } else {
955 pci_pm_reenable_device(pci_dev);
956 }
957
958 return 0;
959}
960
961#else /* !CONFIG_SUSPEND */
962
963#define pci_pm_suspend NULL
964#define pci_pm_suspend_late NULL
965#define pci_pm_suspend_noirq NULL
966#define pci_pm_resume NULL
967#define pci_pm_resume_early NULL
968#define pci_pm_resume_noirq NULL
969
970#endif /* !CONFIG_SUSPEND */
971
972#ifdef CONFIG_HIBERNATE_CALLBACKS
973
974static int pci_pm_freeze(struct device *dev)
975{
976 struct pci_dev *pci_dev = to_pci_dev(dev);
977 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
978
979 if (pci_has_legacy_pm_support(pci_dev))
980 return pci_legacy_suspend(dev, PMSG_FREEZE);
981
982 if (!pm) {
983 pci_pm_default_suspend(pci_dev);
984 return 0;
985 }
986
987 /*
988 * Resume all runtime-suspended devices before creating a snapshot
989 * image of system memory, because the restore kernel generally cannot
990 * be expected to always handle them consistently and they need to be
991 * put into the runtime-active metastate during system resume anyway,
992 * so it is better to ensure that the state saved in the image will be
993 * always consistent with that.
994 */
995 pm_runtime_resume(dev);
996 pci_dev->state_saved = false;
997
998 if (pm->freeze) {
999 int error;
1000
1001 error = pm->freeze(dev);
1002 suspend_report_result(pm->freeze, error);
1003 if (error)
1004 return error;
1005 }
1006
1007 return 0;
1008}
1009
1010static int pci_pm_freeze_noirq(struct device *dev)
1011{
1012 struct pci_dev *pci_dev = to_pci_dev(dev);
1013 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1014
1015 if (pci_has_legacy_pm_support(pci_dev))
1016 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
1017
1018 if (pm && pm->freeze_noirq) {
1019 int error;
1020
1021 error = pm->freeze_noirq(dev);
1022 suspend_report_result(pm->freeze_noirq, error);
1023 if (error)
1024 return error;
1025 }
1026
1027 if (!pci_dev->state_saved)
1028 pci_save_state(pci_dev);
1029
1030 pci_pm_set_unknown_state(pci_dev);
1031
1032 return 0;
1033}
1034
1035static int pci_pm_thaw_noirq(struct device *dev)
1036{
1037 struct pci_dev *pci_dev = to_pci_dev(dev);
1038 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1039
1040 /*
1041 * The pm->thaw_noirq() callback assumes the device has been
1042 * returned to D0 and its config state has been restored.
1043 *
1044 * In addition, pci_restore_state() restores MSI-X state in MMIO
1045 * space, which requires the device to be in D0, so return it to D0
1046 * in case the driver's "freeze" callbacks put it into a low-power
1047 * state.
1048 */
1049 pci_set_power_state(pci_dev, PCI_D0);
1050 pci_restore_state(pci_dev);
1051
1052 if (pci_has_legacy_pm_support(pci_dev))
1053 return 0;
1054
1055 if (pm && pm->thaw_noirq)
1056 return pm->thaw_noirq(dev);
1057
1058 return 0;
1059}
1060
1061static int pci_pm_thaw(struct device *dev)
1062{
1063 struct pci_dev *pci_dev = to_pci_dev(dev);
1064 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1065 int error = 0;
1066
1067 if (pci_has_legacy_pm_support(pci_dev))
1068 return pci_legacy_resume(dev);
1069
1070 if (pm) {
1071 if (pm->thaw)
1072 error = pm->thaw(dev);
1073 } else {
1074 pci_pm_reenable_device(pci_dev);
1075 }
1076
1077 pci_dev->state_saved = false;
1078
1079 return error;
1080}
1081
1082static int pci_pm_poweroff(struct device *dev)
1083{
1084 struct pci_dev *pci_dev = to_pci_dev(dev);
1085 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1086
1087 if (pci_has_legacy_pm_support(pci_dev))
1088 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1089
1090 if (!pm) {
1091 pci_pm_default_suspend(pci_dev);
1092 return 0;
1093 }
1094
1095 /* The reason to do that is the same as in pci_pm_suspend(). */
1096 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1097 pci_dev_need_resume(pci_dev)) {
1098 pm_runtime_resume(dev);
1099 pci_dev->state_saved = false;
1100 } else {
1101 pci_dev_adjust_pme(pci_dev);
1102 }
1103
1104 if (pm->poweroff) {
1105 int error;
1106
1107 error = pm->poweroff(dev);
1108 suspend_report_result(pm->poweroff, error);
1109 if (error)
1110 return error;
1111 }
1112
1113 return 0;
1114}
1115
1116static int pci_pm_poweroff_late(struct device *dev)
1117{
1118 if (dev_pm_skip_suspend(dev))
1119 return 0;
1120
1121 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
1122
1123 return pm_generic_poweroff_late(dev);
1124}
1125
1126static int pci_pm_poweroff_noirq(struct device *dev)
1127{
1128 struct pci_dev *pci_dev = to_pci_dev(dev);
1129 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1130
1131 if (dev_pm_skip_suspend(dev))
1132 return 0;
1133
1134 if (pci_has_legacy_pm_support(pci_dev))
1135 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
1136
1137 if (!pm) {
1138 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1139 return 0;
1140 }
1141
1142 if (pm->poweroff_noirq) {
1143 int error;
1144
1145 error = pm->poweroff_noirq(dev);
1146 suspend_report_result(pm->poweroff_noirq, error);
1147 if (error)
1148 return error;
1149 }
1150
1151 if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1152 pci_prepare_to_sleep(pci_dev);
1153
1154 /*
1155 * The reason for doing this here is the same as for the analogous code
1156 * in pci_pm_suspend_noirq().
1157 */
1158 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1159 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1160
1161 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1162
1163 return 0;
1164}
1165
1166static int pci_pm_restore_noirq(struct device *dev)
1167{
1168 struct pci_dev *pci_dev = to_pci_dev(dev);
1169 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1170
1171 pci_pm_default_resume_early(pci_dev);
1172 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1173
1174 if (pci_has_legacy_pm_support(pci_dev))
1175 return 0;
1176
1177 if (pm && pm->restore_noirq)
1178 return pm->restore_noirq(dev);
1179
1180 return 0;
1181}
1182
1183static int pci_pm_restore(struct device *dev)
1184{
1185 struct pci_dev *pci_dev = to_pci_dev(dev);
1186 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1187
1188 /*
1189 * This is necessary for the hibernation error path in which restore is
1190 * called without restoring the standard config registers of the device.
1191 */
1192 if (pci_dev->state_saved)
1193 pci_restore_standard_config(pci_dev);
1194
1195 if (pci_has_legacy_pm_support(pci_dev))
1196 return pci_legacy_resume(dev);
1197
1198 pci_pm_default_resume(pci_dev);
1199
1200 if (pm) {
1201 if (pm->restore)
1202 return pm->restore(dev);
1203 } else {
1204 pci_pm_reenable_device(pci_dev);
1205 }
1206
1207 return 0;
1208}
1209
1210#else /* !CONFIG_HIBERNATE_CALLBACKS */
1211
1212#define pci_pm_freeze NULL
1213#define pci_pm_freeze_noirq NULL
1214#define pci_pm_thaw NULL
1215#define pci_pm_thaw_noirq NULL
1216#define pci_pm_poweroff NULL
1217#define pci_pm_poweroff_late NULL
1218#define pci_pm_poweroff_noirq NULL
1219#define pci_pm_restore NULL
1220#define pci_pm_restore_noirq NULL
1221
1222#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1223
1224#ifdef CONFIG_PM
1225
1226static int pci_pm_runtime_suspend(struct device *dev)
1227{
1228 struct pci_dev *pci_dev = to_pci_dev(dev);
1229 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1230 pci_power_t prev = pci_dev->current_state;
1231 int error;
1232
1233 /*
1234 * If pci_dev->driver is not set (unbound), we leave the device in D0,
1235 * but it may go to D3cold when the bridge above it runtime suspends.
1236 * Save its config space in case that happens.
1237 */
1238 if (!pci_dev->driver) {
1239 pci_save_state(pci_dev);
1240 return 0;
1241 }
1242
1243 pci_dev->state_saved = false;
1244 if (pm && pm->runtime_suspend) {
1245 error = pm->runtime_suspend(dev);
1246 /*
1247 * -EBUSY and -EAGAIN is used to request the runtime PM core
1248 * to schedule a new suspend, so log the event only with debug
1249 * log level.
1250 */
1251 if (error == -EBUSY || error == -EAGAIN) {
1252 pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n",
1253 pm->runtime_suspend, error);
1254 return error;
1255 } else if (error) {
1256 pci_err(pci_dev, "can't suspend (%ps returned %d)\n",
1257 pm->runtime_suspend, error);
1258 return error;
1259 }
1260 }
1261
1262 pci_fixup_device(pci_fixup_suspend, pci_dev);
1263
1264 if (pm && pm->runtime_suspend
1265 && !pci_dev->state_saved && pci_dev->current_state != PCI_D0
1266 && pci_dev->current_state != PCI_UNKNOWN) {
1267 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
1268 "PCI PM: State of device not saved by %pS\n",
1269 pm->runtime_suspend);
1270 return 0;
1271 }
1272
1273 if (!pci_dev->state_saved) {
1274 pci_save_state(pci_dev);
1275 pci_finish_runtime_suspend(pci_dev);
1276 }
1277
1278 return 0;
1279}
1280
1281static int pci_pm_runtime_resume(struct device *dev)
1282{
1283 struct pci_dev *pci_dev = to_pci_dev(dev);
1284 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1285 pci_power_t prev_state = pci_dev->current_state;
1286 int error = 0;
1287
1288 /*
1289 * Restoring config space is necessary even if the device is not bound
1290 * to a driver because although we left it in D0, it may have gone to
1291 * D3cold when the bridge above it runtime suspended.
1292 */
1293 pci_restore_standard_config(pci_dev);
1294
1295 if (!pci_dev->driver)
1296 return 0;
1297
1298 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1299 pci_pm_default_resume(pci_dev);
1300
1301 if (prev_state == PCI_D3cold)
1302 pci_bridge_wait_for_secondary_bus(pci_dev);
1303
1304 if (pm && pm->runtime_resume)
1305 error = pm->runtime_resume(dev);
1306
1307 pci_dev->runtime_d3cold = false;
1308
1309 return error;
1310}
1311
1312static int pci_pm_runtime_idle(struct device *dev)
1313{
1314 struct pci_dev *pci_dev = to_pci_dev(dev);
1315 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1316
1317 /*
1318 * If pci_dev->driver is not set (unbound), the device should
1319 * always remain in D0 regardless of the runtime PM status
1320 */
1321 if (!pci_dev->driver)
1322 return 0;
1323
1324 if (!pm)
1325 return -ENOSYS;
1326
1327 if (pm->runtime_idle)
1328 return pm->runtime_idle(dev);
1329
1330 return 0;
1331}
1332
1333static const struct dev_pm_ops pci_dev_pm_ops = {
1334 .prepare = pci_pm_prepare,
1335 .complete = pci_pm_complete,
1336 .suspend = pci_pm_suspend,
1337 .suspend_late = pci_pm_suspend_late,
1338 .resume = pci_pm_resume,
1339 .resume_early = pci_pm_resume_early,
1340 .freeze = pci_pm_freeze,
1341 .thaw = pci_pm_thaw,
1342 .poweroff = pci_pm_poweroff,
1343 .poweroff_late = pci_pm_poweroff_late,
1344 .restore = pci_pm_restore,
1345 .suspend_noirq = pci_pm_suspend_noirq,
1346 .resume_noirq = pci_pm_resume_noirq,
1347 .freeze_noirq = pci_pm_freeze_noirq,
1348 .thaw_noirq = pci_pm_thaw_noirq,
1349 .poweroff_noirq = pci_pm_poweroff_noirq,
1350 .restore_noirq = pci_pm_restore_noirq,
1351 .runtime_suspend = pci_pm_runtime_suspend,
1352 .runtime_resume = pci_pm_runtime_resume,
1353 .runtime_idle = pci_pm_runtime_idle,
1354};
1355
1356#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1357
1358#else /* !CONFIG_PM */
1359
1360#define pci_pm_runtime_suspend NULL
1361#define pci_pm_runtime_resume NULL
1362#define pci_pm_runtime_idle NULL
1363
1364#define PCI_PM_OPS_PTR NULL
1365
1366#endif /* !CONFIG_PM */
1367
1368/**
1369 * __pci_register_driver - register a new pci driver
1370 * @drv: the driver structure to register
1371 * @owner: owner module of drv
1372 * @mod_name: module name string
1373 *
1374 * Adds the driver structure to the list of registered drivers.
1375 * Returns a negative value on error, otherwise 0.
1376 * If no error occurred, the driver remains registered even if
1377 * no device was claimed during registration.
1378 */
1379int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1380 const char *mod_name)
1381{
1382 /* initialize common driver fields */
1383 drv->driver.name = drv->name;
1384 drv->driver.bus = &pci_bus_type;
1385 drv->driver.owner = owner;
1386 drv->driver.mod_name = mod_name;
1387 drv->driver.groups = drv->groups;
1388 drv->driver.dev_groups = drv->dev_groups;
1389
1390 spin_lock_init(&drv->dynids.lock);
1391 INIT_LIST_HEAD(&drv->dynids.list);
1392
1393 /* register with core */
1394 return driver_register(&drv->driver);
1395}
1396EXPORT_SYMBOL(__pci_register_driver);
1397
1398/**
1399 * pci_unregister_driver - unregister a pci driver
1400 * @drv: the driver structure to unregister
1401 *
1402 * Deletes the driver structure from the list of registered PCI drivers,
1403 * gives it a chance to clean up by calling its remove() function for
1404 * each device it was responsible for, and marks those devices as
1405 * driverless.
1406 */
1407
1408void pci_unregister_driver(struct pci_driver *drv)
1409{
1410 driver_unregister(&drv->driver);
1411 pci_free_dynids(drv);
1412}
1413EXPORT_SYMBOL(pci_unregister_driver);
1414
1415static struct pci_driver pci_compat_driver = {
1416 .name = "compat"
1417};
1418
1419/**
1420 * pci_dev_driver - get the pci_driver of a device
1421 * @dev: the device to query
1422 *
1423 * Returns the appropriate pci_driver structure or %NULL if there is no
1424 * registered driver for the device.
1425 */
1426struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1427{
1428 if (dev->driver)
1429 return dev->driver;
1430 else {
1431 int i;
1432 for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1433 if (dev->resource[i].flags & IORESOURCE_BUSY)
1434 return &pci_compat_driver;
1435 }
1436 return NULL;
1437}
1438EXPORT_SYMBOL(pci_dev_driver);
1439
1440/**
1441 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1442 * @dev: the PCI device structure to match against
1443 * @drv: the device driver to search for matching PCI device id structures
1444 *
1445 * Used by a driver to check whether a PCI device present in the
1446 * system is in its list of supported devices. Returns the matching
1447 * pci_device_id structure or %NULL if there is no match.
1448 */
1449static int pci_bus_match(struct device *dev, struct device_driver *drv)
1450{
1451 struct pci_dev *pci_dev = to_pci_dev(dev);
1452 struct pci_driver *pci_drv;
1453 const struct pci_device_id *found_id;
1454
1455 if (!pci_dev->match_driver)
1456 return 0;
1457
1458 pci_drv = to_pci_driver(drv);
1459 found_id = pci_match_device(pci_drv, pci_dev);
1460 if (found_id)
1461 return 1;
1462
1463 return 0;
1464}
1465
1466/**
1467 * pci_dev_get - increments the reference count of the pci device structure
1468 * @dev: the device being referenced
1469 *
1470 * Each live reference to a device should be refcounted.
1471 *
1472 * Drivers for PCI devices should normally record such references in
1473 * their probe() methods, when they bind to a device, and release
1474 * them by calling pci_dev_put(), in their disconnect() methods.
1475 *
1476 * A pointer to the device with the incremented reference counter is returned.
1477 */
1478struct pci_dev *pci_dev_get(struct pci_dev *dev)
1479{
1480 if (dev)
1481 get_device(&dev->dev);
1482 return dev;
1483}
1484EXPORT_SYMBOL(pci_dev_get);
1485
1486/**
1487 * pci_dev_put - release a use of the pci device structure
1488 * @dev: device that's been disconnected
1489 *
1490 * Must be called when a user of a device is finished with it. When the last
1491 * user of the device calls this function, the memory of the device is freed.
1492 */
1493void pci_dev_put(struct pci_dev *dev)
1494{
1495 if (dev)
1496 put_device(&dev->dev);
1497}
1498EXPORT_SYMBOL(pci_dev_put);
1499
1500static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1501{
1502 struct pci_dev *pdev;
1503
1504 if (!dev)
1505 return -ENODEV;
1506
1507 pdev = to_pci_dev(dev);
1508
1509 if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1510 return -ENOMEM;
1511
1512 if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1513 return -ENOMEM;
1514
1515 if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1516 pdev->subsystem_device))
1517 return -ENOMEM;
1518
1519 if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1520 return -ENOMEM;
1521
1522 if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1523 pdev->vendor, pdev->device,
1524 pdev->subsystem_vendor, pdev->subsystem_device,
1525 (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1526 (u8)(pdev->class)))
1527 return -ENOMEM;
1528
1529 return 0;
1530}
1531
1532#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH)
1533/**
1534 * pci_uevent_ers - emit a uevent during recovery path of PCI device
1535 * @pdev: PCI device undergoing error recovery
1536 * @err_type: type of error event
1537 */
1538void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1539{
1540 int idx = 0;
1541 char *envp[3];
1542
1543 switch (err_type) {
1544 case PCI_ERS_RESULT_NONE:
1545 case PCI_ERS_RESULT_CAN_RECOVER:
1546 envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1547 envp[idx++] = "DEVICE_ONLINE=0";
1548 break;
1549 case PCI_ERS_RESULT_RECOVERED:
1550 envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1551 envp[idx++] = "DEVICE_ONLINE=1";
1552 break;
1553 case PCI_ERS_RESULT_DISCONNECT:
1554 envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1555 envp[idx++] = "DEVICE_ONLINE=0";
1556 break;
1557 default:
1558 break;
1559 }
1560
1561 if (idx > 0) {
1562 envp[idx++] = NULL;
1563 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1564 }
1565}
1566#endif
1567
1568static int pci_bus_num_vf(struct device *dev)
1569{
1570 return pci_num_vf(to_pci_dev(dev));
1571}
1572
1573/**
1574 * pci_dma_configure - Setup DMA configuration
1575 * @dev: ptr to dev structure
1576 *
1577 * Function to update PCI devices's DMA configuration using the same
1578 * info from the OF node or ACPI node of host bridge's parent (if any).
1579 */
1580static int pci_dma_configure(struct device *dev)
1581{
1582 struct device *bridge;
1583 int ret = 0;
1584
1585 bridge = pci_get_host_bridge_device(to_pci_dev(dev));
1586
1587 if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
1588 bridge->parent->of_node) {
1589 ret = of_dma_configure(dev, bridge->parent->of_node, true);
1590 } else if (has_acpi_companion(bridge)) {
1591 struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
1592
1593 ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev));
1594 }
1595
1596 pci_put_host_bridge_device(bridge);
1597 return ret;
1598}
1599
1600struct bus_type pci_bus_type = {
1601 .name = "pci",
1602 .match = pci_bus_match,
1603 .uevent = pci_uevent,
1604 .probe = pci_device_probe,
1605 .remove = pci_device_remove,
1606 .shutdown = pci_device_shutdown,
1607 .dev_groups = pci_dev_groups,
1608 .bus_groups = pci_bus_groups,
1609 .drv_groups = pci_drv_groups,
1610 .pm = PCI_PM_OPS_PTR,
1611 .num_vf = pci_bus_num_vf,
1612 .dma_configure = pci_dma_configure,
1613};
1614EXPORT_SYMBOL(pci_bus_type);
1615
1616#ifdef CONFIG_PCIEPORTBUS
1617static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
1618{
1619 struct pcie_device *pciedev;
1620 struct pcie_port_service_driver *driver;
1621
1622 if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1623 return 0;
1624
1625 pciedev = to_pcie_device(dev);
1626 driver = to_service_driver(drv);
1627
1628 if (driver->service != pciedev->service)
1629 return 0;
1630
1631 if (driver->port_type != PCIE_ANY_PORT &&
1632 driver->port_type != pci_pcie_type(pciedev->port))
1633 return 0;
1634
1635 return 1;
1636}
1637
1638struct bus_type pcie_port_bus_type = {
1639 .name = "pci_express",
1640 .match = pcie_port_bus_match,
1641};
1642EXPORT_SYMBOL_GPL(pcie_port_bus_type);
1643#endif
1644
1645static int __init pci_driver_init(void)
1646{
1647 int ret;
1648
1649 ret = bus_register(&pci_bus_type);
1650 if (ret)
1651 return ret;
1652
1653#ifdef CONFIG_PCIEPORTBUS
1654 ret = bus_register(&pcie_port_bus_type);
1655 if (ret)
1656 return ret;
1657#endif
1658 dma_debug_add_bus(&pci_bus_type);
1659 return 0;
1660}
1661postcore_initcall(pci_driver_init);