Loading...
1/*
2 * drivers/pci/pci-driver.c
3 *
4 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
5 * (C) Copyright 2007 Novell Inc.
6 *
7 * Released under the GPL v2 only.
8 *
9 */
10
11#include <linux/pci.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/device.h>
15#include <linux/mempolicy.h>
16#include <linux/string.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/cpu.h>
20#include <linux/pm_runtime.h>
21#include <linux/suspend.h>
22#include "pci.h"
23
24struct pci_dynid {
25 struct list_head node;
26 struct pci_device_id id;
27};
28
29/**
30 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
31 * @drv: target pci driver
32 * @vendor: PCI vendor ID
33 * @device: PCI device ID
34 * @subvendor: PCI subvendor ID
35 * @subdevice: PCI subdevice ID
36 * @class: PCI class
37 * @class_mask: PCI class mask
38 * @driver_data: private driver data
39 *
40 * Adds a new dynamic pci device ID to this driver and causes the
41 * driver to probe for all devices again. @drv must have been
42 * registered prior to calling this function.
43 *
44 * CONTEXT:
45 * Does GFP_KERNEL allocation.
46 *
47 * RETURNS:
48 * 0 on success, -errno on failure.
49 */
50int pci_add_dynid(struct pci_driver *drv,
51 unsigned int vendor, unsigned int device,
52 unsigned int subvendor, unsigned int subdevice,
53 unsigned int class, unsigned int class_mask,
54 unsigned long driver_data)
55{
56 struct pci_dynid *dynid;
57 int retval;
58
59 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
60 if (!dynid)
61 return -ENOMEM;
62
63 dynid->id.vendor = vendor;
64 dynid->id.device = device;
65 dynid->id.subvendor = subvendor;
66 dynid->id.subdevice = subdevice;
67 dynid->id.class = class;
68 dynid->id.class_mask = class_mask;
69 dynid->id.driver_data = driver_data;
70
71 spin_lock(&drv->dynids.lock);
72 list_add_tail(&dynid->node, &drv->dynids.list);
73 spin_unlock(&drv->dynids.lock);
74
75 get_driver(&drv->driver);
76 retval = driver_attach(&drv->driver);
77 put_driver(&drv->driver);
78
79 return retval;
80}
81
82static void pci_free_dynids(struct pci_driver *drv)
83{
84 struct pci_dynid *dynid, *n;
85
86 spin_lock(&drv->dynids.lock);
87 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
88 list_del(&dynid->node);
89 kfree(dynid);
90 }
91 spin_unlock(&drv->dynids.lock);
92}
93
94/*
95 * Dynamic device ID manipulation via sysfs is disabled for !CONFIG_HOTPLUG
96 */
97#ifdef CONFIG_HOTPLUG
98/**
99 * store_new_id - sysfs frontend to pci_add_dynid()
100 * @driver: target device driver
101 * @buf: buffer for scanning device ID data
102 * @count: input size
103 *
104 * Allow PCI IDs to be added to an existing driver via sysfs.
105 */
106static ssize_t
107store_new_id(struct device_driver *driver, const char *buf, size_t count)
108{
109 struct pci_driver *pdrv = to_pci_driver(driver);
110 const struct pci_device_id *ids = pdrv->id_table;
111 __u32 vendor, device, subvendor=PCI_ANY_ID,
112 subdevice=PCI_ANY_ID, class=0, class_mask=0;
113 unsigned long driver_data=0;
114 int fields=0;
115 int retval;
116
117 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
118 &vendor, &device, &subvendor, &subdevice,
119 &class, &class_mask, &driver_data);
120 if (fields < 2)
121 return -EINVAL;
122
123 /* Only accept driver_data values that match an existing id_table
124 entry */
125 if (ids) {
126 retval = -EINVAL;
127 while (ids->vendor || ids->subvendor || ids->class_mask) {
128 if (driver_data == ids->driver_data) {
129 retval = 0;
130 break;
131 }
132 ids++;
133 }
134 if (retval) /* No match */
135 return retval;
136 }
137
138 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
139 class, class_mask, driver_data);
140 if (retval)
141 return retval;
142 return count;
143}
144static DRIVER_ATTR(new_id, S_IWUSR, NULL, store_new_id);
145
146/**
147 * store_remove_id - remove a PCI device ID from this driver
148 * @driver: target device driver
149 * @buf: buffer for scanning device ID data
150 * @count: input size
151 *
152 * Removes a dynamic pci device ID to this driver.
153 */
154static ssize_t
155store_remove_id(struct device_driver *driver, const char *buf, size_t count)
156{
157 struct pci_dynid *dynid, *n;
158 struct pci_driver *pdrv = to_pci_driver(driver);
159 __u32 vendor, device, subvendor = PCI_ANY_ID,
160 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
161 int fields = 0;
162 int retval = -ENODEV;
163
164 fields = sscanf(buf, "%x %x %x %x %x %x",
165 &vendor, &device, &subvendor, &subdevice,
166 &class, &class_mask);
167 if (fields < 2)
168 return -EINVAL;
169
170 spin_lock(&pdrv->dynids.lock);
171 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
172 struct pci_device_id *id = &dynid->id;
173 if ((id->vendor == vendor) &&
174 (id->device == device) &&
175 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
176 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
177 !((id->class ^ class) & class_mask)) {
178 list_del(&dynid->node);
179 kfree(dynid);
180 retval = 0;
181 break;
182 }
183 }
184 spin_unlock(&pdrv->dynids.lock);
185
186 if (retval)
187 return retval;
188 return count;
189}
190static DRIVER_ATTR(remove_id, S_IWUSR, NULL, store_remove_id);
191
192static int
193pci_create_newid_file(struct pci_driver *drv)
194{
195 int error = 0;
196 if (drv->probe != NULL)
197 error = driver_create_file(&drv->driver, &driver_attr_new_id);
198 return error;
199}
200
201static void pci_remove_newid_file(struct pci_driver *drv)
202{
203 driver_remove_file(&drv->driver, &driver_attr_new_id);
204}
205
206static int
207pci_create_removeid_file(struct pci_driver *drv)
208{
209 int error = 0;
210 if (drv->probe != NULL)
211 error = driver_create_file(&drv->driver,&driver_attr_remove_id);
212 return error;
213}
214
215static void pci_remove_removeid_file(struct pci_driver *drv)
216{
217 driver_remove_file(&drv->driver, &driver_attr_remove_id);
218}
219#else /* !CONFIG_HOTPLUG */
220static inline int pci_create_newid_file(struct pci_driver *drv)
221{
222 return 0;
223}
224static inline void pci_remove_newid_file(struct pci_driver *drv) {}
225static inline int pci_create_removeid_file(struct pci_driver *drv)
226{
227 return 0;
228}
229static inline void pci_remove_removeid_file(struct pci_driver *drv) {}
230#endif
231
232/**
233 * pci_match_id - See if a pci device matches a given pci_id table
234 * @ids: array of PCI device id structures to search in
235 * @dev: the PCI device structure to match against.
236 *
237 * Used by a driver to check whether a PCI device present in the
238 * system is in its list of supported devices. Returns the matching
239 * pci_device_id structure or %NULL if there is no match.
240 *
241 * Deprecated, don't use this as it will not catch any dynamic ids
242 * that a driver might want to check for.
243 */
244const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
245 struct pci_dev *dev)
246{
247 if (ids) {
248 while (ids->vendor || ids->subvendor || ids->class_mask) {
249 if (pci_match_one_device(ids, dev))
250 return ids;
251 ids++;
252 }
253 }
254 return NULL;
255}
256
257/**
258 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
259 * @drv: the PCI driver to match against
260 * @dev: the PCI device structure to match against
261 *
262 * Used by a driver to check whether a PCI device present in the
263 * system is in its list of supported devices. Returns the matching
264 * pci_device_id structure or %NULL if there is no match.
265 */
266static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
267 struct pci_dev *dev)
268{
269 struct pci_dynid *dynid;
270
271 /* Look at the dynamic ids first, before the static ones */
272 spin_lock(&drv->dynids.lock);
273 list_for_each_entry(dynid, &drv->dynids.list, node) {
274 if (pci_match_one_device(&dynid->id, dev)) {
275 spin_unlock(&drv->dynids.lock);
276 return &dynid->id;
277 }
278 }
279 spin_unlock(&drv->dynids.lock);
280
281 return pci_match_id(drv->id_table, dev);
282}
283
284struct drv_dev_and_id {
285 struct pci_driver *drv;
286 struct pci_dev *dev;
287 const struct pci_device_id *id;
288};
289
290static long local_pci_probe(void *_ddi)
291{
292 struct drv_dev_and_id *ddi = _ddi;
293 struct device *dev = &ddi->dev->dev;
294 int rc;
295
296 /* Unbound PCI devices are always set to disabled and suspended.
297 * During probe, the device is set to enabled and active and the
298 * usage count is incremented. If the driver supports runtime PM,
299 * it should call pm_runtime_put_noidle() in its probe routine and
300 * pm_runtime_get_noresume() in its remove routine.
301 */
302 pm_runtime_get_noresume(dev);
303 pm_runtime_set_active(dev);
304 pm_runtime_enable(dev);
305
306 rc = ddi->drv->probe(ddi->dev, ddi->id);
307 if (rc) {
308 pm_runtime_disable(dev);
309 pm_runtime_set_suspended(dev);
310 pm_runtime_put_noidle(dev);
311 }
312 return rc;
313}
314
315static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
316 const struct pci_device_id *id)
317{
318 int error, node;
319 struct drv_dev_and_id ddi = { drv, dev, id };
320
321 /* Execute driver initialization on node where the device's
322 bus is attached to. This way the driver likely allocates
323 its local memory on the right node without any need to
324 change it. */
325 node = dev_to_node(&dev->dev);
326 if (node >= 0) {
327 int cpu;
328
329 get_online_cpus();
330 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
331 if (cpu < nr_cpu_ids)
332 error = work_on_cpu(cpu, local_pci_probe, &ddi);
333 else
334 error = local_pci_probe(&ddi);
335 put_online_cpus();
336 } else
337 error = local_pci_probe(&ddi);
338 return error;
339}
340
341/**
342 * __pci_device_probe - check if a driver wants to claim a specific PCI device
343 * @drv: driver to call to check if it wants the PCI device
344 * @pci_dev: PCI device being probed
345 *
346 * returns 0 on success, else error.
347 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
348 */
349static int
350__pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
351{
352 const struct pci_device_id *id;
353 int error = 0;
354
355 if (!pci_dev->driver && drv->probe) {
356 error = -ENODEV;
357
358 id = pci_match_device(drv, pci_dev);
359 if (id)
360 error = pci_call_probe(drv, pci_dev, id);
361 if (error >= 0) {
362 pci_dev->driver = drv;
363 error = 0;
364 }
365 }
366 return error;
367}
368
369static int pci_device_probe(struct device * dev)
370{
371 int error = 0;
372 struct pci_driver *drv;
373 struct pci_dev *pci_dev;
374
375 drv = to_pci_driver(dev->driver);
376 pci_dev = to_pci_dev(dev);
377 pci_dev_get(pci_dev);
378 error = __pci_device_probe(drv, pci_dev);
379 if (error)
380 pci_dev_put(pci_dev);
381
382 return error;
383}
384
385static int pci_device_remove(struct device * dev)
386{
387 struct pci_dev * pci_dev = to_pci_dev(dev);
388 struct pci_driver * drv = pci_dev->driver;
389
390 if (drv) {
391 if (drv->remove) {
392 pm_runtime_get_sync(dev);
393 drv->remove(pci_dev);
394 pm_runtime_put_noidle(dev);
395 }
396 pci_dev->driver = NULL;
397 }
398
399 /* Undo the runtime PM settings in local_pci_probe() */
400 pm_runtime_disable(dev);
401 pm_runtime_set_suspended(dev);
402 pm_runtime_put_noidle(dev);
403
404 /*
405 * If the device is still on, set the power state as "unknown",
406 * since it might change by the next time we load the driver.
407 */
408 if (pci_dev->current_state == PCI_D0)
409 pci_dev->current_state = PCI_UNKNOWN;
410
411 /*
412 * We would love to complain here if pci_dev->is_enabled is set, that
413 * the driver should have called pci_disable_device(), but the
414 * unfortunate fact is there are too many odd BIOS and bridge setups
415 * that don't like drivers doing that all of the time.
416 * Oh well, we can dream of sane hardware when we sleep, no matter how
417 * horrible the crap we have to deal with is when we are awake...
418 */
419
420 pci_dev_put(pci_dev);
421 return 0;
422}
423
424static void pci_device_shutdown(struct device *dev)
425{
426 struct pci_dev *pci_dev = to_pci_dev(dev);
427 struct pci_driver *drv = pci_dev->driver;
428
429 if (drv && drv->shutdown)
430 drv->shutdown(pci_dev);
431 pci_msi_shutdown(pci_dev);
432 pci_msix_shutdown(pci_dev);
433}
434
435#ifdef CONFIG_PM
436
437/* Auxiliary functions used for system resume and run-time resume. */
438
439/**
440 * pci_restore_standard_config - restore standard config registers of PCI device
441 * @pci_dev: PCI device to handle
442 */
443static int pci_restore_standard_config(struct pci_dev *pci_dev)
444{
445 pci_update_current_state(pci_dev, PCI_UNKNOWN);
446
447 if (pci_dev->current_state != PCI_D0) {
448 int error = pci_set_power_state(pci_dev, PCI_D0);
449 if (error)
450 return error;
451 }
452
453 pci_restore_state(pci_dev);
454 return 0;
455}
456
457static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
458{
459 pci_restore_standard_config(pci_dev);
460 pci_fixup_device(pci_fixup_resume_early, pci_dev);
461}
462
463#endif
464
465#ifdef CONFIG_PM_SLEEP
466
467/*
468 * Default "suspend" method for devices that have no driver provided suspend,
469 * or not even a driver at all (second part).
470 */
471static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
472{
473 /*
474 * mark its power state as "unknown", since we don't know if
475 * e.g. the BIOS will change its device state when we suspend.
476 */
477 if (pci_dev->current_state == PCI_D0)
478 pci_dev->current_state = PCI_UNKNOWN;
479}
480
481/*
482 * Default "resume" method for devices that have no driver provided resume,
483 * or not even a driver at all (second part).
484 */
485static int pci_pm_reenable_device(struct pci_dev *pci_dev)
486{
487 int retval;
488
489 /* if the device was enabled before suspend, reenable */
490 retval = pci_reenable_device(pci_dev);
491 /*
492 * if the device was busmaster before the suspend, make it busmaster
493 * again
494 */
495 if (pci_dev->is_busmaster)
496 pci_set_master(pci_dev);
497
498 return retval;
499}
500
501static int pci_legacy_suspend(struct device *dev, pm_message_t state)
502{
503 struct pci_dev * pci_dev = to_pci_dev(dev);
504 struct pci_driver * drv = pci_dev->driver;
505
506 if (drv && drv->suspend) {
507 pci_power_t prev = pci_dev->current_state;
508 int error;
509
510 error = drv->suspend(pci_dev, state);
511 suspend_report_result(drv->suspend, error);
512 if (error)
513 return error;
514
515 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
516 && pci_dev->current_state != PCI_UNKNOWN) {
517 WARN_ONCE(pci_dev->current_state != prev,
518 "PCI PM: Device state not saved by %pF\n",
519 drv->suspend);
520 }
521 }
522
523 pci_fixup_device(pci_fixup_suspend, pci_dev);
524
525 return 0;
526}
527
528static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
529{
530 struct pci_dev * pci_dev = to_pci_dev(dev);
531 struct pci_driver * drv = pci_dev->driver;
532
533 if (drv && drv->suspend_late) {
534 pci_power_t prev = pci_dev->current_state;
535 int error;
536
537 error = drv->suspend_late(pci_dev, state);
538 suspend_report_result(drv->suspend_late, error);
539 if (error)
540 return error;
541
542 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
543 && pci_dev->current_state != PCI_UNKNOWN) {
544 WARN_ONCE(pci_dev->current_state != prev,
545 "PCI PM: Device state not saved by %pF\n",
546 drv->suspend_late);
547 return 0;
548 }
549 }
550
551 if (!pci_dev->state_saved)
552 pci_save_state(pci_dev);
553
554 pci_pm_set_unknown_state(pci_dev);
555
556 return 0;
557}
558
559static int pci_legacy_resume_early(struct device *dev)
560{
561 struct pci_dev * pci_dev = to_pci_dev(dev);
562 struct pci_driver * drv = pci_dev->driver;
563
564 return drv && drv->resume_early ?
565 drv->resume_early(pci_dev) : 0;
566}
567
568static int pci_legacy_resume(struct device *dev)
569{
570 struct pci_dev * pci_dev = to_pci_dev(dev);
571 struct pci_driver * drv = pci_dev->driver;
572
573 pci_fixup_device(pci_fixup_resume, pci_dev);
574
575 return drv && drv->resume ?
576 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
577}
578
579/* Auxiliary functions used by the new power management framework */
580
581static void pci_pm_default_resume(struct pci_dev *pci_dev)
582{
583 pci_fixup_device(pci_fixup_resume, pci_dev);
584
585 if (!pci_is_bridge(pci_dev))
586 pci_enable_wake(pci_dev, PCI_D0, false);
587}
588
589static void pci_pm_default_suspend(struct pci_dev *pci_dev)
590{
591 /* Disable non-bridge devices without PM support */
592 if (!pci_is_bridge(pci_dev))
593 pci_disable_enabled_device(pci_dev);
594}
595
596static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
597{
598 struct pci_driver *drv = pci_dev->driver;
599 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
600 || drv->resume_early);
601
602 /*
603 * Legacy PM support is used by default, so warn if the new framework is
604 * supported as well. Drivers are supposed to support either the
605 * former, or the latter, but not both at the same time.
606 */
607 WARN_ON(ret && drv->driver.pm);
608
609 return ret;
610}
611
612/* New power management framework */
613
614static int pci_pm_prepare(struct device *dev)
615{
616 struct device_driver *drv = dev->driver;
617 int error = 0;
618
619 /*
620 * If a PCI device configured to wake up the system from sleep states
621 * has been suspended at run time and there's a resume request pending
622 * for it, this is equivalent to the device signaling wakeup, so the
623 * system suspend operation should be aborted.
624 */
625 pm_runtime_get_noresume(dev);
626 if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
627 pm_wakeup_event(dev, 0);
628
629 if (pm_wakeup_pending()) {
630 pm_runtime_put_sync(dev);
631 return -EBUSY;
632 }
633
634 /*
635 * PCI devices suspended at run time need to be resumed at this
636 * point, because in general it is necessary to reconfigure them for
637 * system suspend. Namely, if the device is supposed to wake up the
638 * system from the sleep state, we may need to reconfigure it for this
639 * purpose. In turn, if the device is not supposed to wake up the
640 * system from the sleep state, we'll have to prevent it from signaling
641 * wake-up.
642 */
643 pm_runtime_resume(dev);
644
645 if (drv && drv->pm && drv->pm->prepare)
646 error = drv->pm->prepare(dev);
647
648 return error;
649}
650
651static void pci_pm_complete(struct device *dev)
652{
653 struct device_driver *drv = dev->driver;
654
655 if (drv && drv->pm && drv->pm->complete)
656 drv->pm->complete(dev);
657
658 pm_runtime_put_sync(dev);
659}
660
661#else /* !CONFIG_PM_SLEEP */
662
663#define pci_pm_prepare NULL
664#define pci_pm_complete NULL
665
666#endif /* !CONFIG_PM_SLEEP */
667
668#ifdef CONFIG_SUSPEND
669
670static int pci_pm_suspend(struct device *dev)
671{
672 struct pci_dev *pci_dev = to_pci_dev(dev);
673 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
674
675 if (pci_has_legacy_pm_support(pci_dev))
676 return pci_legacy_suspend(dev, PMSG_SUSPEND);
677
678 if (!pm) {
679 pci_pm_default_suspend(pci_dev);
680 goto Fixup;
681 }
682
683 if (pm->suspend) {
684 pci_power_t prev = pci_dev->current_state;
685 int error;
686
687 error = pm->suspend(dev);
688 suspend_report_result(pm->suspend, error);
689 if (error)
690 return error;
691
692 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
693 && pci_dev->current_state != PCI_UNKNOWN) {
694 WARN_ONCE(pci_dev->current_state != prev,
695 "PCI PM: State of device not saved by %pF\n",
696 pm->suspend);
697 }
698 }
699
700 Fixup:
701 pci_fixup_device(pci_fixup_suspend, pci_dev);
702
703 return 0;
704}
705
706static int pci_pm_suspend_noirq(struct device *dev)
707{
708 struct pci_dev *pci_dev = to_pci_dev(dev);
709 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
710
711 if (pci_has_legacy_pm_support(pci_dev))
712 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
713
714 if (!pm) {
715 pci_save_state(pci_dev);
716 return 0;
717 }
718
719 if (pm->suspend_noirq) {
720 pci_power_t prev = pci_dev->current_state;
721 int error;
722
723 error = pm->suspend_noirq(dev);
724 suspend_report_result(pm->suspend_noirq, error);
725 if (error)
726 return error;
727
728 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
729 && pci_dev->current_state != PCI_UNKNOWN) {
730 WARN_ONCE(pci_dev->current_state != prev,
731 "PCI PM: State of device not saved by %pF\n",
732 pm->suspend_noirq);
733 return 0;
734 }
735 }
736
737 if (!pci_dev->state_saved) {
738 pci_save_state(pci_dev);
739 if (!pci_is_bridge(pci_dev))
740 pci_prepare_to_sleep(pci_dev);
741 }
742
743 pci_pm_set_unknown_state(pci_dev);
744
745 return 0;
746}
747
748static int pci_pm_resume_noirq(struct device *dev)
749{
750 struct pci_dev *pci_dev = to_pci_dev(dev);
751 struct device_driver *drv = dev->driver;
752 int error = 0;
753
754 pci_pm_default_resume_early(pci_dev);
755
756 if (pci_has_legacy_pm_support(pci_dev))
757 return pci_legacy_resume_early(dev);
758
759 if (drv && drv->pm && drv->pm->resume_noirq)
760 error = drv->pm->resume_noirq(dev);
761
762 return error;
763}
764
765static int pci_pm_resume(struct device *dev)
766{
767 struct pci_dev *pci_dev = to_pci_dev(dev);
768 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
769 int error = 0;
770
771 /*
772 * This is necessary for the suspend error path in which resume is
773 * called without restoring the standard config registers of the device.
774 */
775 if (pci_dev->state_saved)
776 pci_restore_standard_config(pci_dev);
777
778 if (pci_has_legacy_pm_support(pci_dev))
779 return pci_legacy_resume(dev);
780
781 pci_pm_default_resume(pci_dev);
782
783 if (pm) {
784 if (pm->resume)
785 error = pm->resume(dev);
786 } else {
787 pci_pm_reenable_device(pci_dev);
788 }
789
790 return error;
791}
792
793#else /* !CONFIG_SUSPEND */
794
795#define pci_pm_suspend NULL
796#define pci_pm_suspend_noirq NULL
797#define pci_pm_resume NULL
798#define pci_pm_resume_noirq NULL
799
800#endif /* !CONFIG_SUSPEND */
801
802#ifdef CONFIG_HIBERNATE_CALLBACKS
803
804static int pci_pm_freeze(struct device *dev)
805{
806 struct pci_dev *pci_dev = to_pci_dev(dev);
807 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
808
809 if (pci_has_legacy_pm_support(pci_dev))
810 return pci_legacy_suspend(dev, PMSG_FREEZE);
811
812 if (!pm) {
813 pci_pm_default_suspend(pci_dev);
814 return 0;
815 }
816
817 if (pm->freeze) {
818 int error;
819
820 error = pm->freeze(dev);
821 suspend_report_result(pm->freeze, error);
822 if (error)
823 return error;
824 }
825
826 return 0;
827}
828
829static int pci_pm_freeze_noirq(struct device *dev)
830{
831 struct pci_dev *pci_dev = to_pci_dev(dev);
832 struct device_driver *drv = dev->driver;
833
834 if (pci_has_legacy_pm_support(pci_dev))
835 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
836
837 if (drv && drv->pm && drv->pm->freeze_noirq) {
838 int error;
839
840 error = drv->pm->freeze_noirq(dev);
841 suspend_report_result(drv->pm->freeze_noirq, error);
842 if (error)
843 return error;
844 }
845
846 if (!pci_dev->state_saved)
847 pci_save_state(pci_dev);
848
849 pci_pm_set_unknown_state(pci_dev);
850
851 return 0;
852}
853
854static int pci_pm_thaw_noirq(struct device *dev)
855{
856 struct pci_dev *pci_dev = to_pci_dev(dev);
857 struct device_driver *drv = dev->driver;
858 int error = 0;
859
860 if (pci_has_legacy_pm_support(pci_dev))
861 return pci_legacy_resume_early(dev);
862
863 pci_update_current_state(pci_dev, PCI_D0);
864
865 if (drv && drv->pm && drv->pm->thaw_noirq)
866 error = drv->pm->thaw_noirq(dev);
867
868 return error;
869}
870
871static int pci_pm_thaw(struct device *dev)
872{
873 struct pci_dev *pci_dev = to_pci_dev(dev);
874 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
875 int error = 0;
876
877 if (pci_has_legacy_pm_support(pci_dev))
878 return pci_legacy_resume(dev);
879
880 if (pm) {
881 if (pm->thaw)
882 error = pm->thaw(dev);
883 } else {
884 pci_pm_reenable_device(pci_dev);
885 }
886
887 pci_dev->state_saved = false;
888
889 return error;
890}
891
892static int pci_pm_poweroff(struct device *dev)
893{
894 struct pci_dev *pci_dev = to_pci_dev(dev);
895 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
896
897 if (pci_has_legacy_pm_support(pci_dev))
898 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
899
900 if (!pm) {
901 pci_pm_default_suspend(pci_dev);
902 goto Fixup;
903 }
904
905 if (pm->poweroff) {
906 int error;
907
908 error = pm->poweroff(dev);
909 suspend_report_result(pm->poweroff, error);
910 if (error)
911 return error;
912 }
913
914 Fixup:
915 pci_fixup_device(pci_fixup_suspend, pci_dev);
916
917 return 0;
918}
919
920static int pci_pm_poweroff_noirq(struct device *dev)
921{
922 struct pci_dev *pci_dev = to_pci_dev(dev);
923 struct device_driver *drv = dev->driver;
924
925 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
926 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
927
928 if (!drv || !drv->pm)
929 return 0;
930
931 if (drv->pm->poweroff_noirq) {
932 int error;
933
934 error = drv->pm->poweroff_noirq(dev);
935 suspend_report_result(drv->pm->poweroff_noirq, error);
936 if (error)
937 return error;
938 }
939
940 if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
941 pci_prepare_to_sleep(pci_dev);
942
943 return 0;
944}
945
946static int pci_pm_restore_noirq(struct device *dev)
947{
948 struct pci_dev *pci_dev = to_pci_dev(dev);
949 struct device_driver *drv = dev->driver;
950 int error = 0;
951
952 pci_pm_default_resume_early(pci_dev);
953
954 if (pci_has_legacy_pm_support(pci_dev))
955 return pci_legacy_resume_early(dev);
956
957 if (drv && drv->pm && drv->pm->restore_noirq)
958 error = drv->pm->restore_noirq(dev);
959
960 return error;
961}
962
963static int pci_pm_restore(struct device *dev)
964{
965 struct pci_dev *pci_dev = to_pci_dev(dev);
966 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
967 int error = 0;
968
969 /*
970 * This is necessary for the hibernation error path in which restore is
971 * called without restoring the standard config registers of the device.
972 */
973 if (pci_dev->state_saved)
974 pci_restore_standard_config(pci_dev);
975
976 if (pci_has_legacy_pm_support(pci_dev))
977 return pci_legacy_resume(dev);
978
979 pci_pm_default_resume(pci_dev);
980
981 if (pm) {
982 if (pm->restore)
983 error = pm->restore(dev);
984 } else {
985 pci_pm_reenable_device(pci_dev);
986 }
987
988 return error;
989}
990
991#else /* !CONFIG_HIBERNATE_CALLBACKS */
992
993#define pci_pm_freeze NULL
994#define pci_pm_freeze_noirq NULL
995#define pci_pm_thaw NULL
996#define pci_pm_thaw_noirq NULL
997#define pci_pm_poweroff NULL
998#define pci_pm_poweroff_noirq NULL
999#define pci_pm_restore NULL
1000#define pci_pm_restore_noirq NULL
1001
1002#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1003
1004#ifdef CONFIG_PM_RUNTIME
1005
1006static int pci_pm_runtime_suspend(struct device *dev)
1007{
1008 struct pci_dev *pci_dev = to_pci_dev(dev);
1009 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1010 pci_power_t prev = pci_dev->current_state;
1011 int error;
1012
1013 if (!pm || !pm->runtime_suspend)
1014 return -ENOSYS;
1015
1016 error = pm->runtime_suspend(dev);
1017 suspend_report_result(pm->runtime_suspend, error);
1018 if (error)
1019 return error;
1020
1021 pci_fixup_device(pci_fixup_suspend, pci_dev);
1022
1023 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1024 && pci_dev->current_state != PCI_UNKNOWN) {
1025 WARN_ONCE(pci_dev->current_state != prev,
1026 "PCI PM: State of device not saved by %pF\n",
1027 pm->runtime_suspend);
1028 return 0;
1029 }
1030
1031 if (!pci_dev->state_saved)
1032 pci_save_state(pci_dev);
1033
1034 pci_finish_runtime_suspend(pci_dev);
1035
1036 return 0;
1037}
1038
1039static int pci_pm_runtime_resume(struct device *dev)
1040{
1041 struct pci_dev *pci_dev = to_pci_dev(dev);
1042 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1043
1044 if (!pm || !pm->runtime_resume)
1045 return -ENOSYS;
1046
1047 pci_pm_default_resume_early(pci_dev);
1048 __pci_enable_wake(pci_dev, PCI_D0, true, false);
1049 pci_fixup_device(pci_fixup_resume, pci_dev);
1050
1051 return pm->runtime_resume(dev);
1052}
1053
1054static int pci_pm_runtime_idle(struct device *dev)
1055{
1056 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1057
1058 if (!pm)
1059 return -ENOSYS;
1060
1061 if (pm->runtime_idle) {
1062 int ret = pm->runtime_idle(dev);
1063 if (ret)
1064 return ret;
1065 }
1066
1067 pm_runtime_suspend(dev);
1068
1069 return 0;
1070}
1071
1072#else /* !CONFIG_PM_RUNTIME */
1073
1074#define pci_pm_runtime_suspend NULL
1075#define pci_pm_runtime_resume NULL
1076#define pci_pm_runtime_idle NULL
1077
1078#endif /* !CONFIG_PM_RUNTIME */
1079
1080#ifdef CONFIG_PM
1081
1082const struct dev_pm_ops pci_dev_pm_ops = {
1083 .prepare = pci_pm_prepare,
1084 .complete = pci_pm_complete,
1085 .suspend = pci_pm_suspend,
1086 .resume = pci_pm_resume,
1087 .freeze = pci_pm_freeze,
1088 .thaw = pci_pm_thaw,
1089 .poweroff = pci_pm_poweroff,
1090 .restore = pci_pm_restore,
1091 .suspend_noirq = pci_pm_suspend_noirq,
1092 .resume_noirq = pci_pm_resume_noirq,
1093 .freeze_noirq = pci_pm_freeze_noirq,
1094 .thaw_noirq = pci_pm_thaw_noirq,
1095 .poweroff_noirq = pci_pm_poweroff_noirq,
1096 .restore_noirq = pci_pm_restore_noirq,
1097 .runtime_suspend = pci_pm_runtime_suspend,
1098 .runtime_resume = pci_pm_runtime_resume,
1099 .runtime_idle = pci_pm_runtime_idle,
1100};
1101
1102#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1103
1104#else /* !COMFIG_PM_OPS */
1105
1106#define PCI_PM_OPS_PTR NULL
1107
1108#endif /* !COMFIG_PM_OPS */
1109
1110/**
1111 * __pci_register_driver - register a new pci driver
1112 * @drv: the driver structure to register
1113 * @owner: owner module of drv
1114 * @mod_name: module name string
1115 *
1116 * Adds the driver structure to the list of registered drivers.
1117 * Returns a negative value on error, otherwise 0.
1118 * If no error occurred, the driver remains registered even if
1119 * no device was claimed during registration.
1120 */
1121int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1122 const char *mod_name)
1123{
1124 int error;
1125
1126 /* initialize common driver fields */
1127 drv->driver.name = drv->name;
1128 drv->driver.bus = &pci_bus_type;
1129 drv->driver.owner = owner;
1130 drv->driver.mod_name = mod_name;
1131
1132 spin_lock_init(&drv->dynids.lock);
1133 INIT_LIST_HEAD(&drv->dynids.list);
1134
1135 /* register with core */
1136 error = driver_register(&drv->driver);
1137 if (error)
1138 goto out;
1139
1140 error = pci_create_newid_file(drv);
1141 if (error)
1142 goto out_newid;
1143
1144 error = pci_create_removeid_file(drv);
1145 if (error)
1146 goto out_removeid;
1147out:
1148 return error;
1149
1150out_removeid:
1151 pci_remove_newid_file(drv);
1152out_newid:
1153 driver_unregister(&drv->driver);
1154 goto out;
1155}
1156
1157/**
1158 * pci_unregister_driver - unregister a pci driver
1159 * @drv: the driver structure to unregister
1160 *
1161 * Deletes the driver structure from the list of registered PCI drivers,
1162 * gives it a chance to clean up by calling its remove() function for
1163 * each device it was responsible for, and marks those devices as
1164 * driverless.
1165 */
1166
1167void
1168pci_unregister_driver(struct pci_driver *drv)
1169{
1170 pci_remove_removeid_file(drv);
1171 pci_remove_newid_file(drv);
1172 driver_unregister(&drv->driver);
1173 pci_free_dynids(drv);
1174}
1175
1176static struct pci_driver pci_compat_driver = {
1177 .name = "compat"
1178};
1179
1180/**
1181 * pci_dev_driver - get the pci_driver of a device
1182 * @dev: the device to query
1183 *
1184 * Returns the appropriate pci_driver structure or %NULL if there is no
1185 * registered driver for the device.
1186 */
1187struct pci_driver *
1188pci_dev_driver(const struct pci_dev *dev)
1189{
1190 if (dev->driver)
1191 return dev->driver;
1192 else {
1193 int i;
1194 for(i=0; i<=PCI_ROM_RESOURCE; i++)
1195 if (dev->resource[i].flags & IORESOURCE_BUSY)
1196 return &pci_compat_driver;
1197 }
1198 return NULL;
1199}
1200
1201/**
1202 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1203 * @dev: the PCI device structure to match against
1204 * @drv: the device driver to search for matching PCI device id structures
1205 *
1206 * Used by a driver to check whether a PCI device present in the
1207 * system is in its list of supported devices. Returns the matching
1208 * pci_device_id structure or %NULL if there is no match.
1209 */
1210static int pci_bus_match(struct device *dev, struct device_driver *drv)
1211{
1212 struct pci_dev *pci_dev = to_pci_dev(dev);
1213 struct pci_driver *pci_drv = to_pci_driver(drv);
1214 const struct pci_device_id *found_id;
1215
1216 found_id = pci_match_device(pci_drv, pci_dev);
1217 if (found_id)
1218 return 1;
1219
1220 return 0;
1221}
1222
1223/**
1224 * pci_dev_get - increments the reference count of the pci device structure
1225 * @dev: the device being referenced
1226 *
1227 * Each live reference to a device should be refcounted.
1228 *
1229 * Drivers for PCI devices should normally record such references in
1230 * their probe() methods, when they bind to a device, and release
1231 * them by calling pci_dev_put(), in their disconnect() methods.
1232 *
1233 * A pointer to the device with the incremented reference counter is returned.
1234 */
1235struct pci_dev *pci_dev_get(struct pci_dev *dev)
1236{
1237 if (dev)
1238 get_device(&dev->dev);
1239 return dev;
1240}
1241
1242/**
1243 * pci_dev_put - release a use of the pci device structure
1244 * @dev: device that's been disconnected
1245 *
1246 * Must be called when a user of a device is finished with it. When the last
1247 * user of the device calls this function, the memory of the device is freed.
1248 */
1249void pci_dev_put(struct pci_dev *dev)
1250{
1251 if (dev)
1252 put_device(&dev->dev);
1253}
1254
1255#ifndef CONFIG_HOTPLUG
1256int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1257{
1258 return -ENODEV;
1259}
1260#endif
1261
1262struct bus_type pci_bus_type = {
1263 .name = "pci",
1264 .match = pci_bus_match,
1265 .uevent = pci_uevent,
1266 .probe = pci_device_probe,
1267 .remove = pci_device_remove,
1268 .shutdown = pci_device_shutdown,
1269 .dev_attrs = pci_dev_attrs,
1270 .bus_attrs = pci_bus_attrs,
1271 .pm = PCI_PM_OPS_PTR,
1272};
1273
1274static int __init pci_driver_init(void)
1275{
1276 return bus_register(&pci_bus_type);
1277}
1278
1279postcore_initcall(pci_driver_init);
1280
1281EXPORT_SYMBOL_GPL(pci_add_dynid);
1282EXPORT_SYMBOL(pci_match_id);
1283EXPORT_SYMBOL(__pci_register_driver);
1284EXPORT_SYMBOL(pci_unregister_driver);
1285EXPORT_SYMBOL(pci_dev_driver);
1286EXPORT_SYMBOL(pci_bus_type);
1287EXPORT_SYMBOL(pci_dev_get);
1288EXPORT_SYMBOL(pci_dev_put);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
4 * (C) Copyright 2007 Novell Inc.
5 */
6
7#include <linux/pci.h>
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/device.h>
11#include <linux/mempolicy.h>
12#include <linux/string.h>
13#include <linux/slab.h>
14#include <linux/sched.h>
15#include <linux/sched/isolation.h>
16#include <linux/cpu.h>
17#include <linux/pm_runtime.h>
18#include <linux/suspend.h>
19#include <linux/kexec.h>
20#include <linux/of_device.h>
21#include <linux/acpi.h>
22#include <linux/dma-map-ops.h>
23#include <linux/iommu.h>
24#include "pci.h"
25#include "pcie/portdrv.h"
26
27struct pci_dynid {
28 struct list_head node;
29 struct pci_device_id id;
30};
31
32/**
33 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
34 * @drv: target pci driver
35 * @vendor: PCI vendor ID
36 * @device: PCI device ID
37 * @subvendor: PCI subvendor ID
38 * @subdevice: PCI subdevice ID
39 * @class: PCI class
40 * @class_mask: PCI class mask
41 * @driver_data: private driver data
42 *
43 * Adds a new dynamic pci device ID to this driver and causes the
44 * driver to probe for all devices again. @drv must have been
45 * registered prior to calling this function.
46 *
47 * CONTEXT:
48 * Does GFP_KERNEL allocation.
49 *
50 * RETURNS:
51 * 0 on success, -errno on failure.
52 */
53int pci_add_dynid(struct pci_driver *drv,
54 unsigned int vendor, unsigned int device,
55 unsigned int subvendor, unsigned int subdevice,
56 unsigned int class, unsigned int class_mask,
57 unsigned long driver_data)
58{
59 struct pci_dynid *dynid;
60
61 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
62 if (!dynid)
63 return -ENOMEM;
64
65 dynid->id.vendor = vendor;
66 dynid->id.device = device;
67 dynid->id.subvendor = subvendor;
68 dynid->id.subdevice = subdevice;
69 dynid->id.class = class;
70 dynid->id.class_mask = class_mask;
71 dynid->id.driver_data = driver_data;
72
73 spin_lock(&drv->dynids.lock);
74 list_add_tail(&dynid->node, &drv->dynids.list);
75 spin_unlock(&drv->dynids.lock);
76
77 return driver_attach(&drv->driver);
78}
79EXPORT_SYMBOL_GPL(pci_add_dynid);
80
81static void pci_free_dynids(struct pci_driver *drv)
82{
83 struct pci_dynid *dynid, *n;
84
85 spin_lock(&drv->dynids.lock);
86 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
87 list_del(&dynid->node);
88 kfree(dynid);
89 }
90 spin_unlock(&drv->dynids.lock);
91}
92
93/**
94 * pci_match_id - See if a PCI device matches a given pci_id table
95 * @ids: array of PCI device ID structures to search in
96 * @dev: the PCI device structure to match against.
97 *
98 * Used by a driver to check whether a PCI device is in its list of
99 * supported devices. Returns the matching pci_device_id structure or
100 * %NULL if there is no match.
101 *
102 * Deprecated; don't use this as it will not catch any dynamic IDs
103 * that a driver might want to check for.
104 */
105const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
106 struct pci_dev *dev)
107{
108 if (ids) {
109 while (ids->vendor || ids->subvendor || ids->class_mask) {
110 if (pci_match_one_device(ids, dev))
111 return ids;
112 ids++;
113 }
114 }
115 return NULL;
116}
117EXPORT_SYMBOL(pci_match_id);
118
119static const struct pci_device_id pci_device_id_any = {
120 .vendor = PCI_ANY_ID,
121 .device = PCI_ANY_ID,
122 .subvendor = PCI_ANY_ID,
123 .subdevice = PCI_ANY_ID,
124};
125
126/**
127 * pci_match_device - See if a device matches a driver's list of IDs
128 * @drv: the PCI driver to match against
129 * @dev: the PCI device structure to match against
130 *
131 * Used by a driver to check whether a PCI device is in its list of
132 * supported devices or in the dynids list, which may have been augmented
133 * via the sysfs "new_id" file. Returns the matching pci_device_id
134 * structure or %NULL if there is no match.
135 */
136static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
137 struct pci_dev *dev)
138{
139 struct pci_dynid *dynid;
140 const struct pci_device_id *found_id = NULL, *ids;
141
142 /* When driver_override is set, only bind to the matching driver */
143 if (dev->driver_override && strcmp(dev->driver_override, drv->name))
144 return NULL;
145
146 /* Look at the dynamic ids first, before the static ones */
147 spin_lock(&drv->dynids.lock);
148 list_for_each_entry(dynid, &drv->dynids.list, node) {
149 if (pci_match_one_device(&dynid->id, dev)) {
150 found_id = &dynid->id;
151 break;
152 }
153 }
154 spin_unlock(&drv->dynids.lock);
155
156 if (found_id)
157 return found_id;
158
159 for (ids = drv->id_table; (found_id = pci_match_id(ids, dev));
160 ids = found_id + 1) {
161 /*
162 * The match table is split based on driver_override.
163 * In case override_only was set, enforce driver_override
164 * matching.
165 */
166 if (found_id->override_only) {
167 if (dev->driver_override)
168 return found_id;
169 } else {
170 return found_id;
171 }
172 }
173
174 /* driver_override will always match, send a dummy id */
175 if (dev->driver_override)
176 return &pci_device_id_any;
177 return NULL;
178}
179
180/**
181 * new_id_store - sysfs frontend to pci_add_dynid()
182 * @driver: target device driver
183 * @buf: buffer for scanning device ID data
184 * @count: input size
185 *
186 * Allow PCI IDs to be added to an existing driver via sysfs.
187 */
188static ssize_t new_id_store(struct device_driver *driver, const char *buf,
189 size_t count)
190{
191 struct pci_driver *pdrv = to_pci_driver(driver);
192 const struct pci_device_id *ids = pdrv->id_table;
193 u32 vendor, device, subvendor = PCI_ANY_ID,
194 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
195 unsigned long driver_data = 0;
196 int fields = 0;
197 int retval = 0;
198
199 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
200 &vendor, &device, &subvendor, &subdevice,
201 &class, &class_mask, &driver_data);
202 if (fields < 2)
203 return -EINVAL;
204
205 if (fields != 7) {
206 struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
207 if (!pdev)
208 return -ENOMEM;
209
210 pdev->vendor = vendor;
211 pdev->device = device;
212 pdev->subsystem_vendor = subvendor;
213 pdev->subsystem_device = subdevice;
214 pdev->class = class;
215
216 if (pci_match_device(pdrv, pdev))
217 retval = -EEXIST;
218
219 kfree(pdev);
220
221 if (retval)
222 return retval;
223 }
224
225 /* Only accept driver_data values that match an existing id_table
226 entry */
227 if (ids) {
228 retval = -EINVAL;
229 while (ids->vendor || ids->subvendor || ids->class_mask) {
230 if (driver_data == ids->driver_data) {
231 retval = 0;
232 break;
233 }
234 ids++;
235 }
236 if (retval) /* No match */
237 return retval;
238 }
239
240 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
241 class, class_mask, driver_data);
242 if (retval)
243 return retval;
244 return count;
245}
246static DRIVER_ATTR_WO(new_id);
247
248/**
249 * remove_id_store - remove a PCI device ID from this driver
250 * @driver: target device driver
251 * @buf: buffer for scanning device ID data
252 * @count: input size
253 *
254 * Removes a dynamic pci device ID to this driver.
255 */
256static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
257 size_t count)
258{
259 struct pci_dynid *dynid, *n;
260 struct pci_driver *pdrv = to_pci_driver(driver);
261 u32 vendor, device, subvendor = PCI_ANY_ID,
262 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
263 int fields = 0;
264 size_t retval = -ENODEV;
265
266 fields = sscanf(buf, "%x %x %x %x %x %x",
267 &vendor, &device, &subvendor, &subdevice,
268 &class, &class_mask);
269 if (fields < 2)
270 return -EINVAL;
271
272 spin_lock(&pdrv->dynids.lock);
273 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
274 struct pci_device_id *id = &dynid->id;
275 if ((id->vendor == vendor) &&
276 (id->device == device) &&
277 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
278 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
279 !((id->class ^ class) & class_mask)) {
280 list_del(&dynid->node);
281 kfree(dynid);
282 retval = count;
283 break;
284 }
285 }
286 spin_unlock(&pdrv->dynids.lock);
287
288 return retval;
289}
290static DRIVER_ATTR_WO(remove_id);
291
292static struct attribute *pci_drv_attrs[] = {
293 &driver_attr_new_id.attr,
294 &driver_attr_remove_id.attr,
295 NULL,
296};
297ATTRIBUTE_GROUPS(pci_drv);
298
299struct drv_dev_and_id {
300 struct pci_driver *drv;
301 struct pci_dev *dev;
302 const struct pci_device_id *id;
303};
304
305static long local_pci_probe(void *_ddi)
306{
307 struct drv_dev_and_id *ddi = _ddi;
308 struct pci_dev *pci_dev = ddi->dev;
309 struct pci_driver *pci_drv = ddi->drv;
310 struct device *dev = &pci_dev->dev;
311 int rc;
312
313 /*
314 * Unbound PCI devices are always put in D0, regardless of
315 * runtime PM status. During probe, the device is set to
316 * active and the usage count is incremented. If the driver
317 * supports runtime PM, it should call pm_runtime_put_noidle(),
318 * or any other runtime PM helper function decrementing the usage
319 * count, in its probe routine and pm_runtime_get_noresume() in
320 * its remove routine.
321 */
322 pm_runtime_get_sync(dev);
323 pci_dev->driver = pci_drv;
324 rc = pci_drv->probe(pci_dev, ddi->id);
325 if (!rc)
326 return rc;
327 if (rc < 0) {
328 pci_dev->driver = NULL;
329 pm_runtime_put_sync(dev);
330 return rc;
331 }
332 /*
333 * Probe function should return < 0 for failure, 0 for success
334 * Treat values > 0 as success, but warn.
335 */
336 pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n",
337 rc);
338 return 0;
339}
340
341static bool pci_physfn_is_probed(struct pci_dev *dev)
342{
343#ifdef CONFIG_PCI_IOV
344 return dev->is_virtfn && dev->physfn->is_probed;
345#else
346 return false;
347#endif
348}
349
350static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
351 const struct pci_device_id *id)
352{
353 int error, node, cpu;
354 struct drv_dev_and_id ddi = { drv, dev, id };
355
356 /*
357 * Execute driver initialization on node where the device is
358 * attached. This way the driver likely allocates its local memory
359 * on the right node.
360 */
361 node = dev_to_node(&dev->dev);
362 dev->is_probed = 1;
363
364 cpu_hotplug_disable();
365
366 /*
367 * Prevent nesting work_on_cpu() for the case where a Virtual Function
368 * device is probed from work_on_cpu() of the Physical device.
369 */
370 if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
371 pci_physfn_is_probed(dev)) {
372 cpu = nr_cpu_ids;
373 } else {
374 cpumask_var_t wq_domain_mask;
375
376 if (!zalloc_cpumask_var(&wq_domain_mask, GFP_KERNEL)) {
377 error = -ENOMEM;
378 goto out;
379 }
380 cpumask_and(wq_domain_mask,
381 housekeeping_cpumask(HK_TYPE_WQ),
382 housekeeping_cpumask(HK_TYPE_DOMAIN));
383
384 cpu = cpumask_any_and(cpumask_of_node(node),
385 wq_domain_mask);
386 free_cpumask_var(wq_domain_mask);
387 }
388
389 if (cpu < nr_cpu_ids)
390 error = work_on_cpu(cpu, local_pci_probe, &ddi);
391 else
392 error = local_pci_probe(&ddi);
393out:
394 dev->is_probed = 0;
395 cpu_hotplug_enable();
396 return error;
397}
398
399/**
400 * __pci_device_probe - check if a driver wants to claim a specific PCI device
401 * @drv: driver to call to check if it wants the PCI device
402 * @pci_dev: PCI device being probed
403 *
404 * returns 0 on success, else error.
405 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
406 */
407static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
408{
409 const struct pci_device_id *id;
410 int error = 0;
411
412 if (drv->probe) {
413 error = -ENODEV;
414
415 id = pci_match_device(drv, pci_dev);
416 if (id)
417 error = pci_call_probe(drv, pci_dev, id);
418 }
419 return error;
420}
421
422int __weak pcibios_alloc_irq(struct pci_dev *dev)
423{
424 return 0;
425}
426
427void __weak pcibios_free_irq(struct pci_dev *dev)
428{
429}
430
431#ifdef CONFIG_PCI_IOV
432static inline bool pci_device_can_probe(struct pci_dev *pdev)
433{
434 return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe ||
435 pdev->driver_override);
436}
437#else
438static inline bool pci_device_can_probe(struct pci_dev *pdev)
439{
440 return true;
441}
442#endif
443
444static int pci_device_probe(struct device *dev)
445{
446 int error;
447 struct pci_dev *pci_dev = to_pci_dev(dev);
448 struct pci_driver *drv = to_pci_driver(dev->driver);
449
450 if (!pci_device_can_probe(pci_dev))
451 return -ENODEV;
452
453 pci_assign_irq(pci_dev);
454
455 error = pcibios_alloc_irq(pci_dev);
456 if (error < 0)
457 return error;
458
459 pci_dev_get(pci_dev);
460 error = __pci_device_probe(drv, pci_dev);
461 if (error) {
462 pcibios_free_irq(pci_dev);
463 pci_dev_put(pci_dev);
464 }
465
466 return error;
467}
468
469static void pci_device_remove(struct device *dev)
470{
471 struct pci_dev *pci_dev = to_pci_dev(dev);
472 struct pci_driver *drv = pci_dev->driver;
473
474 if (drv->remove) {
475 pm_runtime_get_sync(dev);
476 drv->remove(pci_dev);
477 pm_runtime_put_noidle(dev);
478 }
479 pcibios_free_irq(pci_dev);
480 pci_dev->driver = NULL;
481 pci_iov_remove(pci_dev);
482
483 /* Undo the runtime PM settings in local_pci_probe() */
484 pm_runtime_put_sync(dev);
485
486 /*
487 * If the device is still on, set the power state as "unknown",
488 * since it might change by the next time we load the driver.
489 */
490 if (pci_dev->current_state == PCI_D0)
491 pci_dev->current_state = PCI_UNKNOWN;
492
493 /*
494 * We would love to complain here if pci_dev->is_enabled is set, that
495 * the driver should have called pci_disable_device(), but the
496 * unfortunate fact is there are too many odd BIOS and bridge setups
497 * that don't like drivers doing that all of the time.
498 * Oh well, we can dream of sane hardware when we sleep, no matter how
499 * horrible the crap we have to deal with is when we are awake...
500 */
501
502 pci_dev_put(pci_dev);
503}
504
505static void pci_device_shutdown(struct device *dev)
506{
507 struct pci_dev *pci_dev = to_pci_dev(dev);
508 struct pci_driver *drv = pci_dev->driver;
509
510 pm_runtime_resume(dev);
511
512 if (drv && drv->shutdown)
513 drv->shutdown(pci_dev);
514
515 /*
516 * If this is a kexec reboot, turn off Bus Master bit on the
517 * device to tell it to not continue to do DMA. Don't touch
518 * devices in D3cold or unknown states.
519 * If it is not a kexec reboot, firmware will hit the PCI
520 * devices with big hammer and stop their DMA any way.
521 */
522 if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
523 pci_clear_master(pci_dev);
524}
525
526#ifdef CONFIG_PM_SLEEP
527
528/* Auxiliary functions used for system resume */
529
530/**
531 * pci_restore_standard_config - restore standard config registers of PCI device
532 * @pci_dev: PCI device to handle
533 */
534static int pci_restore_standard_config(struct pci_dev *pci_dev)
535{
536 pci_update_current_state(pci_dev, PCI_UNKNOWN);
537
538 if (pci_dev->current_state != PCI_D0) {
539 int error = pci_set_power_state(pci_dev, PCI_D0);
540 if (error)
541 return error;
542 }
543
544 pci_restore_state(pci_dev);
545 pci_pme_restore(pci_dev);
546 return 0;
547}
548#endif /* CONFIG_PM_SLEEP */
549
550#ifdef CONFIG_PM
551
552/* Auxiliary functions used for system resume and run-time resume */
553
554static void pci_pm_default_resume(struct pci_dev *pci_dev)
555{
556 pci_fixup_device(pci_fixup_resume, pci_dev);
557 pci_enable_wake(pci_dev, PCI_D0, false);
558}
559
560static void pci_pm_power_up_and_verify_state(struct pci_dev *pci_dev)
561{
562 pci_power_up(pci_dev);
563 pci_update_current_state(pci_dev, PCI_D0);
564}
565
566static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
567{
568 pci_pm_power_up_and_verify_state(pci_dev);
569 pci_restore_state(pci_dev);
570 pci_pme_restore(pci_dev);
571}
572
573static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev)
574{
575 pci_bridge_wait_for_secondary_bus(pci_dev);
576 /*
577 * When powering on a bridge from D3cold, the whole hierarchy may be
578 * powered on into D0uninitialized state, resume them to give them a
579 * chance to suspend again
580 */
581 pci_resume_bus(pci_dev->subordinate);
582}
583
584#endif /* CONFIG_PM */
585
586#ifdef CONFIG_PM_SLEEP
587
588/*
589 * Default "suspend" method for devices that have no driver provided suspend,
590 * or not even a driver at all (second part).
591 */
592static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
593{
594 /*
595 * mark its power state as "unknown", since we don't know if
596 * e.g. the BIOS will change its device state when we suspend.
597 */
598 if (pci_dev->current_state == PCI_D0)
599 pci_dev->current_state = PCI_UNKNOWN;
600}
601
602/*
603 * Default "resume" method for devices that have no driver provided resume,
604 * or not even a driver at all (second part).
605 */
606static int pci_pm_reenable_device(struct pci_dev *pci_dev)
607{
608 int retval;
609
610 /* if the device was enabled before suspend, re-enable */
611 retval = pci_reenable_device(pci_dev);
612 /*
613 * if the device was busmaster before the suspend, make it busmaster
614 * again
615 */
616 if (pci_dev->is_busmaster)
617 pci_set_master(pci_dev);
618
619 return retval;
620}
621
622static int pci_legacy_suspend(struct device *dev, pm_message_t state)
623{
624 struct pci_dev *pci_dev = to_pci_dev(dev);
625 struct pci_driver *drv = pci_dev->driver;
626
627 if (drv && drv->suspend) {
628 pci_power_t prev = pci_dev->current_state;
629 int error;
630
631 error = drv->suspend(pci_dev, state);
632 suspend_report_result(dev, drv->suspend, error);
633 if (error)
634 return error;
635
636 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
637 && pci_dev->current_state != PCI_UNKNOWN) {
638 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
639 "PCI PM: Device state not saved by %pS\n",
640 drv->suspend);
641 }
642 }
643
644 pci_fixup_device(pci_fixup_suspend, pci_dev);
645
646 return 0;
647}
648
649static int pci_legacy_suspend_late(struct device *dev)
650{
651 struct pci_dev *pci_dev = to_pci_dev(dev);
652
653 if (!pci_dev->state_saved)
654 pci_save_state(pci_dev);
655
656 pci_pm_set_unknown_state(pci_dev);
657
658 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
659
660 return 0;
661}
662
663static int pci_legacy_resume(struct device *dev)
664{
665 struct pci_dev *pci_dev = to_pci_dev(dev);
666 struct pci_driver *drv = pci_dev->driver;
667
668 pci_fixup_device(pci_fixup_resume, pci_dev);
669
670 return drv && drv->resume ?
671 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
672}
673
674/* Auxiliary functions used by the new power management framework */
675
676static void pci_pm_default_suspend(struct pci_dev *pci_dev)
677{
678 /* Disable non-bridge devices without PM support */
679 if (!pci_has_subordinate(pci_dev))
680 pci_disable_enabled_device(pci_dev);
681}
682
683static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
684{
685 struct pci_driver *drv = pci_dev->driver;
686 bool ret = drv && (drv->suspend || drv->resume);
687
688 /*
689 * Legacy PM support is used by default, so warn if the new framework is
690 * supported as well. Drivers are supposed to support either the
691 * former, or the latter, but not both at the same time.
692 */
693 pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n",
694 pci_dev->vendor, pci_dev->device);
695
696 return ret;
697}
698
699/* New power management framework */
700
701static int pci_pm_prepare(struct device *dev)
702{
703 struct pci_dev *pci_dev = to_pci_dev(dev);
704 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
705
706 if (pm && pm->prepare) {
707 int error = pm->prepare(dev);
708 if (error < 0)
709 return error;
710
711 if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
712 return 0;
713 }
714 if (pci_dev_need_resume(pci_dev))
715 return 0;
716
717 /*
718 * The PME setting needs to be adjusted here in case the direct-complete
719 * optimization is used with respect to this device.
720 */
721 pci_dev_adjust_pme(pci_dev);
722 return 1;
723}
724
725static void pci_pm_complete(struct device *dev)
726{
727 struct pci_dev *pci_dev = to_pci_dev(dev);
728
729 pci_dev_complete_resume(pci_dev);
730 pm_generic_complete(dev);
731
732 /* Resume device if platform firmware has put it in reset-power-on */
733 if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
734 pci_power_t pre_sleep_state = pci_dev->current_state;
735
736 pci_refresh_power_state(pci_dev);
737 /*
738 * On platforms with ACPI this check may also trigger for
739 * devices sharing power resources if one of those power
740 * resources has been activated as a result of a change of the
741 * power state of another device sharing it. However, in that
742 * case it is also better to resume the device, in general.
743 */
744 if (pci_dev->current_state < pre_sleep_state)
745 pm_request_resume(dev);
746 }
747}
748
749#else /* !CONFIG_PM_SLEEP */
750
751#define pci_pm_prepare NULL
752#define pci_pm_complete NULL
753
754#endif /* !CONFIG_PM_SLEEP */
755
756#ifdef CONFIG_SUSPEND
757static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
758{
759 /*
760 * Some BIOSes forget to clear Root PME Status bits after system
761 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
762 * Clear those bits now just in case (shouldn't hurt).
763 */
764 if (pci_is_pcie(pci_dev) &&
765 (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
766 pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
767 pcie_clear_root_pme_status(pci_dev);
768}
769
770static int pci_pm_suspend(struct device *dev)
771{
772 struct pci_dev *pci_dev = to_pci_dev(dev);
773 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
774
775 pci_dev->skip_bus_pm = false;
776
777 /*
778 * Disabling PTM allows some systems, e.g., Intel mobile chips
779 * since Coffee Lake, to enter a lower-power PM state.
780 */
781 pci_suspend_ptm(pci_dev);
782
783 if (pci_has_legacy_pm_support(pci_dev))
784 return pci_legacy_suspend(dev, PMSG_SUSPEND);
785
786 if (!pm) {
787 pci_pm_default_suspend(pci_dev);
788 return 0;
789 }
790
791 /*
792 * PCI devices suspended at run time may need to be resumed at this
793 * point, because in general it may be necessary to reconfigure them for
794 * system suspend. Namely, if the device is expected to wake up the
795 * system from the sleep state, it may have to be reconfigured for this
796 * purpose, or if the device is not expected to wake up the system from
797 * the sleep state, it should be prevented from signaling wakeup events
798 * going forward.
799 *
800 * Also if the driver of the device does not indicate that its system
801 * suspend callbacks can cope with runtime-suspended devices, it is
802 * better to resume the device from runtime suspend here.
803 */
804 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
805 pci_dev_need_resume(pci_dev)) {
806 pm_runtime_resume(dev);
807 pci_dev->state_saved = false;
808 } else {
809 pci_dev_adjust_pme(pci_dev);
810 }
811
812 if (pm->suspend) {
813 pci_power_t prev = pci_dev->current_state;
814 int error;
815
816 error = pm->suspend(dev);
817 suspend_report_result(dev, pm->suspend, error);
818 if (error)
819 return error;
820
821 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
822 && pci_dev->current_state != PCI_UNKNOWN) {
823 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
824 "PCI PM: State of device not saved by %pS\n",
825 pm->suspend);
826 }
827 }
828
829 return 0;
830}
831
832static int pci_pm_suspend_late(struct device *dev)
833{
834 if (dev_pm_skip_suspend(dev))
835 return 0;
836
837 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
838
839 return pm_generic_suspend_late(dev);
840}
841
842static int pci_pm_suspend_noirq(struct device *dev)
843{
844 struct pci_dev *pci_dev = to_pci_dev(dev);
845 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
846
847 if (dev_pm_skip_suspend(dev))
848 return 0;
849
850 if (pci_has_legacy_pm_support(pci_dev))
851 return pci_legacy_suspend_late(dev);
852
853 if (!pm) {
854 pci_save_state(pci_dev);
855 goto Fixup;
856 }
857
858 if (pm->suspend_noirq) {
859 pci_power_t prev = pci_dev->current_state;
860 int error;
861
862 error = pm->suspend_noirq(dev);
863 suspend_report_result(dev, pm->suspend_noirq, error);
864 if (error)
865 return error;
866
867 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
868 && pci_dev->current_state != PCI_UNKNOWN) {
869 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
870 "PCI PM: State of device not saved by %pS\n",
871 pm->suspend_noirq);
872 goto Fixup;
873 }
874 }
875
876 if (!pci_dev->state_saved) {
877 pci_save_state(pci_dev);
878
879 /*
880 * If the device is a bridge with a child in D0 below it,
881 * it needs to stay in D0, so check skip_bus_pm to avoid
882 * putting it into a low-power state in that case.
883 */
884 if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev))
885 pci_prepare_to_sleep(pci_dev);
886 }
887
888 pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n",
889 pci_power_name(pci_dev->current_state));
890
891 if (pci_dev->current_state == PCI_D0) {
892 pci_dev->skip_bus_pm = true;
893 /*
894 * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any
895 * downstream device is in D0, so avoid changing the power state
896 * of the parent bridge by setting the skip_bus_pm flag for it.
897 */
898 if (pci_dev->bus->self)
899 pci_dev->bus->self->skip_bus_pm = true;
900 }
901
902 if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) {
903 pci_dbg(pci_dev, "PCI PM: Skipped\n");
904 goto Fixup;
905 }
906
907 pci_pm_set_unknown_state(pci_dev);
908
909 /*
910 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
911 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
912 * hasn't been quiesced and tries to turn it off. If the controller
913 * is already in D3, this can hang or cause memory corruption.
914 *
915 * Since the value of the COMMAND register doesn't matter once the
916 * device has been suspended, we can safely set it to 0 here.
917 */
918 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
919 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
920
921Fixup:
922 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
923
924 /*
925 * If the target system sleep state is suspend-to-idle, it is sufficient
926 * to check whether or not the device's wakeup settings are good for
927 * runtime PM. Otherwise, the pm_resume_via_firmware() check will cause
928 * pci_pm_complete() to take care of fixing up the device's state
929 * anyway, if need be.
930 */
931 if (device_can_wakeup(dev) && !device_may_wakeup(dev))
932 dev->power.may_skip_resume = false;
933
934 return 0;
935}
936
937static int pci_pm_resume_noirq(struct device *dev)
938{
939 struct pci_dev *pci_dev = to_pci_dev(dev);
940 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
941 pci_power_t prev_state = pci_dev->current_state;
942 bool skip_bus_pm = pci_dev->skip_bus_pm;
943
944 if (dev_pm_skip_resume(dev))
945 return 0;
946
947 /*
948 * In the suspend-to-idle case, devices left in D0 during suspend will
949 * stay in D0, so it is not necessary to restore or update their
950 * configuration here and attempting to put them into D0 again is
951 * pointless, so avoid doing that.
952 */
953 if (!(skip_bus_pm && pm_suspend_no_platform()))
954 pci_pm_default_resume_early(pci_dev);
955
956 pci_fixup_device(pci_fixup_resume_early, pci_dev);
957 pcie_pme_root_status_cleanup(pci_dev);
958
959 if (!skip_bus_pm && prev_state == PCI_D3cold)
960 pci_pm_bridge_power_up_actions(pci_dev);
961
962 if (pci_has_legacy_pm_support(pci_dev))
963 return 0;
964
965 if (pm && pm->resume_noirq)
966 return pm->resume_noirq(dev);
967
968 return 0;
969}
970
971static int pci_pm_resume_early(struct device *dev)
972{
973 if (dev_pm_skip_resume(dev))
974 return 0;
975
976 return pm_generic_resume_early(dev);
977}
978
979static int pci_pm_resume(struct device *dev)
980{
981 struct pci_dev *pci_dev = to_pci_dev(dev);
982 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
983
984 /*
985 * This is necessary for the suspend error path in which resume is
986 * called without restoring the standard config registers of the device.
987 */
988 if (pci_dev->state_saved)
989 pci_restore_standard_config(pci_dev);
990
991 pci_resume_ptm(pci_dev);
992
993 if (pci_has_legacy_pm_support(pci_dev))
994 return pci_legacy_resume(dev);
995
996 pci_pm_default_resume(pci_dev);
997
998 if (pm) {
999 if (pm->resume)
1000 return pm->resume(dev);
1001 } else {
1002 pci_pm_reenable_device(pci_dev);
1003 }
1004
1005 return 0;
1006}
1007
1008#else /* !CONFIG_SUSPEND */
1009
1010#define pci_pm_suspend NULL
1011#define pci_pm_suspend_late NULL
1012#define pci_pm_suspend_noirq NULL
1013#define pci_pm_resume NULL
1014#define pci_pm_resume_early NULL
1015#define pci_pm_resume_noirq NULL
1016
1017#endif /* !CONFIG_SUSPEND */
1018
1019#ifdef CONFIG_HIBERNATE_CALLBACKS
1020
1021static int pci_pm_freeze(struct device *dev)
1022{
1023 struct pci_dev *pci_dev = to_pci_dev(dev);
1024 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1025
1026 if (pci_has_legacy_pm_support(pci_dev))
1027 return pci_legacy_suspend(dev, PMSG_FREEZE);
1028
1029 if (!pm) {
1030 pci_pm_default_suspend(pci_dev);
1031 return 0;
1032 }
1033
1034 /*
1035 * Resume all runtime-suspended devices before creating a snapshot
1036 * image of system memory, because the restore kernel generally cannot
1037 * be expected to always handle them consistently and they need to be
1038 * put into the runtime-active metastate during system resume anyway,
1039 * so it is better to ensure that the state saved in the image will be
1040 * always consistent with that.
1041 */
1042 pm_runtime_resume(dev);
1043 pci_dev->state_saved = false;
1044
1045 if (pm->freeze) {
1046 int error;
1047
1048 error = pm->freeze(dev);
1049 suspend_report_result(dev, pm->freeze, error);
1050 if (error)
1051 return error;
1052 }
1053
1054 return 0;
1055}
1056
1057static int pci_pm_freeze_noirq(struct device *dev)
1058{
1059 struct pci_dev *pci_dev = to_pci_dev(dev);
1060 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1061
1062 if (pci_has_legacy_pm_support(pci_dev))
1063 return pci_legacy_suspend_late(dev);
1064
1065 if (pm && pm->freeze_noirq) {
1066 int error;
1067
1068 error = pm->freeze_noirq(dev);
1069 suspend_report_result(dev, pm->freeze_noirq, error);
1070 if (error)
1071 return error;
1072 }
1073
1074 if (!pci_dev->state_saved)
1075 pci_save_state(pci_dev);
1076
1077 pci_pm_set_unknown_state(pci_dev);
1078
1079 return 0;
1080}
1081
1082static int pci_pm_thaw_noirq(struct device *dev)
1083{
1084 struct pci_dev *pci_dev = to_pci_dev(dev);
1085 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1086
1087 /*
1088 * The pm->thaw_noirq() callback assumes the device has been
1089 * returned to D0 and its config state has been restored.
1090 *
1091 * In addition, pci_restore_state() restores MSI-X state in MMIO
1092 * space, which requires the device to be in D0, so return it to D0
1093 * in case the driver's "freeze" callbacks put it into a low-power
1094 * state.
1095 */
1096 pci_pm_power_up_and_verify_state(pci_dev);
1097 pci_restore_state(pci_dev);
1098
1099 if (pci_has_legacy_pm_support(pci_dev))
1100 return 0;
1101
1102 if (pm && pm->thaw_noirq)
1103 return pm->thaw_noirq(dev);
1104
1105 return 0;
1106}
1107
1108static int pci_pm_thaw(struct device *dev)
1109{
1110 struct pci_dev *pci_dev = to_pci_dev(dev);
1111 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1112 int error = 0;
1113
1114 if (pci_has_legacy_pm_support(pci_dev))
1115 return pci_legacy_resume(dev);
1116
1117 if (pm) {
1118 if (pm->thaw)
1119 error = pm->thaw(dev);
1120 } else {
1121 pci_pm_reenable_device(pci_dev);
1122 }
1123
1124 pci_dev->state_saved = false;
1125
1126 return error;
1127}
1128
1129static int pci_pm_poweroff(struct device *dev)
1130{
1131 struct pci_dev *pci_dev = to_pci_dev(dev);
1132 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1133
1134 if (pci_has_legacy_pm_support(pci_dev))
1135 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1136
1137 if (!pm) {
1138 pci_pm_default_suspend(pci_dev);
1139 return 0;
1140 }
1141
1142 /* The reason to do that is the same as in pci_pm_suspend(). */
1143 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1144 pci_dev_need_resume(pci_dev)) {
1145 pm_runtime_resume(dev);
1146 pci_dev->state_saved = false;
1147 } else {
1148 pci_dev_adjust_pme(pci_dev);
1149 }
1150
1151 if (pm->poweroff) {
1152 int error;
1153
1154 error = pm->poweroff(dev);
1155 suspend_report_result(dev, pm->poweroff, error);
1156 if (error)
1157 return error;
1158 }
1159
1160 return 0;
1161}
1162
1163static int pci_pm_poweroff_late(struct device *dev)
1164{
1165 if (dev_pm_skip_suspend(dev))
1166 return 0;
1167
1168 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
1169
1170 return pm_generic_poweroff_late(dev);
1171}
1172
1173static int pci_pm_poweroff_noirq(struct device *dev)
1174{
1175 struct pci_dev *pci_dev = to_pci_dev(dev);
1176 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1177
1178 if (dev_pm_skip_suspend(dev))
1179 return 0;
1180
1181 if (pci_has_legacy_pm_support(pci_dev))
1182 return pci_legacy_suspend_late(dev);
1183
1184 if (!pm) {
1185 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1186 return 0;
1187 }
1188
1189 if (pm->poweroff_noirq) {
1190 int error;
1191
1192 error = pm->poweroff_noirq(dev);
1193 suspend_report_result(dev, pm->poweroff_noirq, error);
1194 if (error)
1195 return error;
1196 }
1197
1198 if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1199 pci_prepare_to_sleep(pci_dev);
1200
1201 /*
1202 * The reason for doing this here is the same as for the analogous code
1203 * in pci_pm_suspend_noirq().
1204 */
1205 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1206 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1207
1208 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1209
1210 return 0;
1211}
1212
1213static int pci_pm_restore_noirq(struct device *dev)
1214{
1215 struct pci_dev *pci_dev = to_pci_dev(dev);
1216 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1217
1218 pci_pm_default_resume_early(pci_dev);
1219 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1220
1221 if (pci_has_legacy_pm_support(pci_dev))
1222 return 0;
1223
1224 if (pm && pm->restore_noirq)
1225 return pm->restore_noirq(dev);
1226
1227 return 0;
1228}
1229
1230static int pci_pm_restore(struct device *dev)
1231{
1232 struct pci_dev *pci_dev = to_pci_dev(dev);
1233 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1234
1235 /*
1236 * This is necessary for the hibernation error path in which restore is
1237 * called without restoring the standard config registers of the device.
1238 */
1239 if (pci_dev->state_saved)
1240 pci_restore_standard_config(pci_dev);
1241
1242 if (pci_has_legacy_pm_support(pci_dev))
1243 return pci_legacy_resume(dev);
1244
1245 pci_pm_default_resume(pci_dev);
1246
1247 if (pm) {
1248 if (pm->restore)
1249 return pm->restore(dev);
1250 } else {
1251 pci_pm_reenable_device(pci_dev);
1252 }
1253
1254 return 0;
1255}
1256
1257#else /* !CONFIG_HIBERNATE_CALLBACKS */
1258
1259#define pci_pm_freeze NULL
1260#define pci_pm_freeze_noirq NULL
1261#define pci_pm_thaw NULL
1262#define pci_pm_thaw_noirq NULL
1263#define pci_pm_poweroff NULL
1264#define pci_pm_poweroff_late NULL
1265#define pci_pm_poweroff_noirq NULL
1266#define pci_pm_restore NULL
1267#define pci_pm_restore_noirq NULL
1268
1269#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1270
1271#ifdef CONFIG_PM
1272
1273static int pci_pm_runtime_suspend(struct device *dev)
1274{
1275 struct pci_dev *pci_dev = to_pci_dev(dev);
1276 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1277 pci_power_t prev = pci_dev->current_state;
1278 int error;
1279
1280 pci_suspend_ptm(pci_dev);
1281
1282 /*
1283 * If pci_dev->driver is not set (unbound), we leave the device in D0,
1284 * but it may go to D3cold when the bridge above it runtime suspends.
1285 * Save its config space in case that happens.
1286 */
1287 if (!pci_dev->driver) {
1288 pci_save_state(pci_dev);
1289 return 0;
1290 }
1291
1292 pci_dev->state_saved = false;
1293 if (pm && pm->runtime_suspend) {
1294 error = pm->runtime_suspend(dev);
1295 /*
1296 * -EBUSY and -EAGAIN is used to request the runtime PM core
1297 * to schedule a new suspend, so log the event only with debug
1298 * log level.
1299 */
1300 if (error == -EBUSY || error == -EAGAIN) {
1301 pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n",
1302 pm->runtime_suspend, error);
1303 return error;
1304 } else if (error) {
1305 pci_err(pci_dev, "can't suspend (%ps returned %d)\n",
1306 pm->runtime_suspend, error);
1307 return error;
1308 }
1309 }
1310
1311 pci_fixup_device(pci_fixup_suspend, pci_dev);
1312
1313 if (pm && pm->runtime_suspend
1314 && !pci_dev->state_saved && pci_dev->current_state != PCI_D0
1315 && pci_dev->current_state != PCI_UNKNOWN) {
1316 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
1317 "PCI PM: State of device not saved by %pS\n",
1318 pm->runtime_suspend);
1319 return 0;
1320 }
1321
1322 if (!pci_dev->state_saved) {
1323 pci_save_state(pci_dev);
1324 pci_finish_runtime_suspend(pci_dev);
1325 }
1326
1327 return 0;
1328}
1329
1330static int pci_pm_runtime_resume(struct device *dev)
1331{
1332 struct pci_dev *pci_dev = to_pci_dev(dev);
1333 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1334 pci_power_t prev_state = pci_dev->current_state;
1335 int error = 0;
1336
1337 /*
1338 * Restoring config space is necessary even if the device is not bound
1339 * to a driver because although we left it in D0, it may have gone to
1340 * D3cold when the bridge above it runtime suspended.
1341 */
1342 pci_pm_default_resume_early(pci_dev);
1343 pci_resume_ptm(pci_dev);
1344
1345 if (!pci_dev->driver)
1346 return 0;
1347
1348 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1349 pci_pm_default_resume(pci_dev);
1350
1351 if (prev_state == PCI_D3cold)
1352 pci_pm_bridge_power_up_actions(pci_dev);
1353
1354 if (pm && pm->runtime_resume)
1355 error = pm->runtime_resume(dev);
1356
1357 return error;
1358}
1359
1360static int pci_pm_runtime_idle(struct device *dev)
1361{
1362 struct pci_dev *pci_dev = to_pci_dev(dev);
1363 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1364
1365 /*
1366 * If pci_dev->driver is not set (unbound), the device should
1367 * always remain in D0 regardless of the runtime PM status
1368 */
1369 if (!pci_dev->driver)
1370 return 0;
1371
1372 if (!pm)
1373 return -ENOSYS;
1374
1375 if (pm->runtime_idle)
1376 return pm->runtime_idle(dev);
1377
1378 return 0;
1379}
1380
1381static const struct dev_pm_ops pci_dev_pm_ops = {
1382 .prepare = pci_pm_prepare,
1383 .complete = pci_pm_complete,
1384 .suspend = pci_pm_suspend,
1385 .suspend_late = pci_pm_suspend_late,
1386 .resume = pci_pm_resume,
1387 .resume_early = pci_pm_resume_early,
1388 .freeze = pci_pm_freeze,
1389 .thaw = pci_pm_thaw,
1390 .poweroff = pci_pm_poweroff,
1391 .poweroff_late = pci_pm_poweroff_late,
1392 .restore = pci_pm_restore,
1393 .suspend_noirq = pci_pm_suspend_noirq,
1394 .resume_noirq = pci_pm_resume_noirq,
1395 .freeze_noirq = pci_pm_freeze_noirq,
1396 .thaw_noirq = pci_pm_thaw_noirq,
1397 .poweroff_noirq = pci_pm_poweroff_noirq,
1398 .restore_noirq = pci_pm_restore_noirq,
1399 .runtime_suspend = pci_pm_runtime_suspend,
1400 .runtime_resume = pci_pm_runtime_resume,
1401 .runtime_idle = pci_pm_runtime_idle,
1402};
1403
1404#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1405
1406#else /* !CONFIG_PM */
1407
1408#define pci_pm_runtime_suspend NULL
1409#define pci_pm_runtime_resume NULL
1410#define pci_pm_runtime_idle NULL
1411
1412#define PCI_PM_OPS_PTR NULL
1413
1414#endif /* !CONFIG_PM */
1415
1416/**
1417 * __pci_register_driver - register a new pci driver
1418 * @drv: the driver structure to register
1419 * @owner: owner module of drv
1420 * @mod_name: module name string
1421 *
1422 * Adds the driver structure to the list of registered drivers.
1423 * Returns a negative value on error, otherwise 0.
1424 * If no error occurred, the driver remains registered even if
1425 * no device was claimed during registration.
1426 */
1427int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1428 const char *mod_name)
1429{
1430 /* initialize common driver fields */
1431 drv->driver.name = drv->name;
1432 drv->driver.bus = &pci_bus_type;
1433 drv->driver.owner = owner;
1434 drv->driver.mod_name = mod_name;
1435 drv->driver.groups = drv->groups;
1436 drv->driver.dev_groups = drv->dev_groups;
1437
1438 spin_lock_init(&drv->dynids.lock);
1439 INIT_LIST_HEAD(&drv->dynids.list);
1440
1441 /* register with core */
1442 return driver_register(&drv->driver);
1443}
1444EXPORT_SYMBOL(__pci_register_driver);
1445
1446/**
1447 * pci_unregister_driver - unregister a pci driver
1448 * @drv: the driver structure to unregister
1449 *
1450 * Deletes the driver structure from the list of registered PCI drivers,
1451 * gives it a chance to clean up by calling its remove() function for
1452 * each device it was responsible for, and marks those devices as
1453 * driverless.
1454 */
1455
1456void pci_unregister_driver(struct pci_driver *drv)
1457{
1458 driver_unregister(&drv->driver);
1459 pci_free_dynids(drv);
1460}
1461EXPORT_SYMBOL(pci_unregister_driver);
1462
1463static struct pci_driver pci_compat_driver = {
1464 .name = "compat"
1465};
1466
1467/**
1468 * pci_dev_driver - get the pci_driver of a device
1469 * @dev: the device to query
1470 *
1471 * Returns the appropriate pci_driver structure or %NULL if there is no
1472 * registered driver for the device.
1473 */
1474struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1475{
1476 if (dev->driver)
1477 return dev->driver;
1478 else {
1479 int i;
1480 for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1481 if (dev->resource[i].flags & IORESOURCE_BUSY)
1482 return &pci_compat_driver;
1483 }
1484 return NULL;
1485}
1486EXPORT_SYMBOL(pci_dev_driver);
1487
1488/**
1489 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1490 * @dev: the PCI device structure to match against
1491 * @drv: the device driver to search for matching PCI device id structures
1492 *
1493 * Used by a driver to check whether a PCI device present in the
1494 * system is in its list of supported devices. Returns the matching
1495 * pci_device_id structure or %NULL if there is no match.
1496 */
1497static int pci_bus_match(struct device *dev, struct device_driver *drv)
1498{
1499 struct pci_dev *pci_dev = to_pci_dev(dev);
1500 struct pci_driver *pci_drv;
1501 const struct pci_device_id *found_id;
1502
1503 if (!pci_dev->match_driver)
1504 return 0;
1505
1506 pci_drv = to_pci_driver(drv);
1507 found_id = pci_match_device(pci_drv, pci_dev);
1508 if (found_id)
1509 return 1;
1510
1511 return 0;
1512}
1513
1514/**
1515 * pci_dev_get - increments the reference count of the pci device structure
1516 * @dev: the device being referenced
1517 *
1518 * Each live reference to a device should be refcounted.
1519 *
1520 * Drivers for PCI devices should normally record such references in
1521 * their probe() methods, when they bind to a device, and release
1522 * them by calling pci_dev_put(), in their disconnect() methods.
1523 *
1524 * A pointer to the device with the incremented reference counter is returned.
1525 */
1526struct pci_dev *pci_dev_get(struct pci_dev *dev)
1527{
1528 if (dev)
1529 get_device(&dev->dev);
1530 return dev;
1531}
1532EXPORT_SYMBOL(pci_dev_get);
1533
1534/**
1535 * pci_dev_put - release a use of the pci device structure
1536 * @dev: device that's been disconnected
1537 *
1538 * Must be called when a user of a device is finished with it. When the last
1539 * user of the device calls this function, the memory of the device is freed.
1540 */
1541void pci_dev_put(struct pci_dev *dev)
1542{
1543 if (dev)
1544 put_device(&dev->dev);
1545}
1546EXPORT_SYMBOL(pci_dev_put);
1547
1548static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1549{
1550 struct pci_dev *pdev;
1551
1552 if (!dev)
1553 return -ENODEV;
1554
1555 pdev = to_pci_dev(dev);
1556
1557 if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1558 return -ENOMEM;
1559
1560 if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1561 return -ENOMEM;
1562
1563 if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1564 pdev->subsystem_device))
1565 return -ENOMEM;
1566
1567 if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1568 return -ENOMEM;
1569
1570 if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1571 pdev->vendor, pdev->device,
1572 pdev->subsystem_vendor, pdev->subsystem_device,
1573 (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1574 (u8)(pdev->class)))
1575 return -ENOMEM;
1576
1577 return 0;
1578}
1579
1580#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH)
1581/**
1582 * pci_uevent_ers - emit a uevent during recovery path of PCI device
1583 * @pdev: PCI device undergoing error recovery
1584 * @err_type: type of error event
1585 */
1586void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1587{
1588 int idx = 0;
1589 char *envp[3];
1590
1591 switch (err_type) {
1592 case PCI_ERS_RESULT_NONE:
1593 case PCI_ERS_RESULT_CAN_RECOVER:
1594 envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1595 envp[idx++] = "DEVICE_ONLINE=0";
1596 break;
1597 case PCI_ERS_RESULT_RECOVERED:
1598 envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1599 envp[idx++] = "DEVICE_ONLINE=1";
1600 break;
1601 case PCI_ERS_RESULT_DISCONNECT:
1602 envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1603 envp[idx++] = "DEVICE_ONLINE=0";
1604 break;
1605 default:
1606 break;
1607 }
1608
1609 if (idx > 0) {
1610 envp[idx++] = NULL;
1611 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1612 }
1613}
1614#endif
1615
1616static int pci_bus_num_vf(struct device *dev)
1617{
1618 return pci_num_vf(to_pci_dev(dev));
1619}
1620
1621/**
1622 * pci_dma_configure - Setup DMA configuration
1623 * @dev: ptr to dev structure
1624 *
1625 * Function to update PCI devices's DMA configuration using the same
1626 * info from the OF node or ACPI node of host bridge's parent (if any).
1627 */
1628static int pci_dma_configure(struct device *dev)
1629{
1630 struct pci_driver *driver = to_pci_driver(dev->driver);
1631 struct device *bridge;
1632 int ret = 0;
1633
1634 bridge = pci_get_host_bridge_device(to_pci_dev(dev));
1635
1636 if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
1637 bridge->parent->of_node) {
1638 ret = of_dma_configure(dev, bridge->parent->of_node, true);
1639 } else if (has_acpi_companion(bridge)) {
1640 struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
1641
1642 ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev));
1643 }
1644
1645 pci_put_host_bridge_device(bridge);
1646
1647 if (!ret && !driver->driver_managed_dma) {
1648 ret = iommu_device_use_default_domain(dev);
1649 if (ret)
1650 arch_teardown_dma_ops(dev);
1651 }
1652
1653 return ret;
1654}
1655
1656static void pci_dma_cleanup(struct device *dev)
1657{
1658 struct pci_driver *driver = to_pci_driver(dev->driver);
1659
1660 if (!driver->driver_managed_dma)
1661 iommu_device_unuse_default_domain(dev);
1662}
1663
1664struct bus_type pci_bus_type = {
1665 .name = "pci",
1666 .match = pci_bus_match,
1667 .uevent = pci_uevent,
1668 .probe = pci_device_probe,
1669 .remove = pci_device_remove,
1670 .shutdown = pci_device_shutdown,
1671 .dev_groups = pci_dev_groups,
1672 .bus_groups = pci_bus_groups,
1673 .drv_groups = pci_drv_groups,
1674 .pm = PCI_PM_OPS_PTR,
1675 .num_vf = pci_bus_num_vf,
1676 .dma_configure = pci_dma_configure,
1677 .dma_cleanup = pci_dma_cleanup,
1678};
1679EXPORT_SYMBOL(pci_bus_type);
1680
1681#ifdef CONFIG_PCIEPORTBUS
1682static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
1683{
1684 struct pcie_device *pciedev;
1685 struct pcie_port_service_driver *driver;
1686
1687 if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1688 return 0;
1689
1690 pciedev = to_pcie_device(dev);
1691 driver = to_service_driver(drv);
1692
1693 if (driver->service != pciedev->service)
1694 return 0;
1695
1696 if (driver->port_type != PCIE_ANY_PORT &&
1697 driver->port_type != pci_pcie_type(pciedev->port))
1698 return 0;
1699
1700 return 1;
1701}
1702
1703struct bus_type pcie_port_bus_type = {
1704 .name = "pci_express",
1705 .match = pcie_port_bus_match,
1706};
1707EXPORT_SYMBOL_GPL(pcie_port_bus_type);
1708#endif
1709
1710static int __init pci_driver_init(void)
1711{
1712 int ret;
1713
1714 ret = bus_register(&pci_bus_type);
1715 if (ret)
1716 return ret;
1717
1718#ifdef CONFIG_PCIEPORTBUS
1719 ret = bus_register(&pcie_port_bus_type);
1720 if (ret)
1721 return ret;
1722#endif
1723 dma_debug_add_bus(&pci_bus_type);
1724 return 0;
1725}
1726postcore_initcall(pci_driver_init);