Loading...
1/*
2 * drivers/pci/pci-driver.c
3 *
4 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
5 * (C) Copyright 2007 Novell Inc.
6 *
7 * Released under the GPL v2 only.
8 *
9 */
10
11#include <linux/pci.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/device.h>
15#include <linux/mempolicy.h>
16#include <linux/string.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/cpu.h>
20#include <linux/pm_runtime.h>
21#include <linux/suspend.h>
22#include "pci.h"
23
24struct pci_dynid {
25 struct list_head node;
26 struct pci_device_id id;
27};
28
29/**
30 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
31 * @drv: target pci driver
32 * @vendor: PCI vendor ID
33 * @device: PCI device ID
34 * @subvendor: PCI subvendor ID
35 * @subdevice: PCI subdevice ID
36 * @class: PCI class
37 * @class_mask: PCI class mask
38 * @driver_data: private driver data
39 *
40 * Adds a new dynamic pci device ID to this driver and causes the
41 * driver to probe for all devices again. @drv must have been
42 * registered prior to calling this function.
43 *
44 * CONTEXT:
45 * Does GFP_KERNEL allocation.
46 *
47 * RETURNS:
48 * 0 on success, -errno on failure.
49 */
50int pci_add_dynid(struct pci_driver *drv,
51 unsigned int vendor, unsigned int device,
52 unsigned int subvendor, unsigned int subdevice,
53 unsigned int class, unsigned int class_mask,
54 unsigned long driver_data)
55{
56 struct pci_dynid *dynid;
57 int retval;
58
59 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
60 if (!dynid)
61 return -ENOMEM;
62
63 dynid->id.vendor = vendor;
64 dynid->id.device = device;
65 dynid->id.subvendor = subvendor;
66 dynid->id.subdevice = subdevice;
67 dynid->id.class = class;
68 dynid->id.class_mask = class_mask;
69 dynid->id.driver_data = driver_data;
70
71 spin_lock(&drv->dynids.lock);
72 list_add_tail(&dynid->node, &drv->dynids.list);
73 spin_unlock(&drv->dynids.lock);
74
75 retval = driver_attach(&drv->driver);
76
77 return retval;
78}
79
80static void pci_free_dynids(struct pci_driver *drv)
81{
82 struct pci_dynid *dynid, *n;
83
84 spin_lock(&drv->dynids.lock);
85 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
86 list_del(&dynid->node);
87 kfree(dynid);
88 }
89 spin_unlock(&drv->dynids.lock);
90}
91
92/*
93 * Dynamic device ID manipulation via sysfs is disabled for !CONFIG_HOTPLUG
94 */
95#ifdef CONFIG_HOTPLUG
96/**
97 * store_new_id - sysfs frontend to pci_add_dynid()
98 * @driver: target device driver
99 * @buf: buffer for scanning device ID data
100 * @count: input size
101 *
102 * Allow PCI IDs to be added to an existing driver via sysfs.
103 */
104static ssize_t
105store_new_id(struct device_driver *driver, const char *buf, size_t count)
106{
107 struct pci_driver *pdrv = to_pci_driver(driver);
108 const struct pci_device_id *ids = pdrv->id_table;
109 __u32 vendor, device, subvendor=PCI_ANY_ID,
110 subdevice=PCI_ANY_ID, class=0, class_mask=0;
111 unsigned long driver_data=0;
112 int fields=0;
113 int retval;
114
115 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
116 &vendor, &device, &subvendor, &subdevice,
117 &class, &class_mask, &driver_data);
118 if (fields < 2)
119 return -EINVAL;
120
121 /* Only accept driver_data values that match an existing id_table
122 entry */
123 if (ids) {
124 retval = -EINVAL;
125 while (ids->vendor || ids->subvendor || ids->class_mask) {
126 if (driver_data == ids->driver_data) {
127 retval = 0;
128 break;
129 }
130 ids++;
131 }
132 if (retval) /* No match */
133 return retval;
134 }
135
136 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
137 class, class_mask, driver_data);
138 if (retval)
139 return retval;
140 return count;
141}
142static DRIVER_ATTR(new_id, S_IWUSR, NULL, store_new_id);
143
144/**
145 * store_remove_id - remove a PCI device ID from this driver
146 * @driver: target device driver
147 * @buf: buffer for scanning device ID data
148 * @count: input size
149 *
150 * Removes a dynamic pci device ID to this driver.
151 */
152static ssize_t
153store_remove_id(struct device_driver *driver, const char *buf, size_t count)
154{
155 struct pci_dynid *dynid, *n;
156 struct pci_driver *pdrv = to_pci_driver(driver);
157 __u32 vendor, device, subvendor = PCI_ANY_ID,
158 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
159 int fields = 0;
160 int retval = -ENODEV;
161
162 fields = sscanf(buf, "%x %x %x %x %x %x",
163 &vendor, &device, &subvendor, &subdevice,
164 &class, &class_mask);
165 if (fields < 2)
166 return -EINVAL;
167
168 spin_lock(&pdrv->dynids.lock);
169 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
170 struct pci_device_id *id = &dynid->id;
171 if ((id->vendor == vendor) &&
172 (id->device == device) &&
173 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
174 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
175 !((id->class ^ class) & class_mask)) {
176 list_del(&dynid->node);
177 kfree(dynid);
178 retval = 0;
179 break;
180 }
181 }
182 spin_unlock(&pdrv->dynids.lock);
183
184 if (retval)
185 return retval;
186 return count;
187}
188static DRIVER_ATTR(remove_id, S_IWUSR, NULL, store_remove_id);
189
190static int
191pci_create_newid_files(struct pci_driver *drv)
192{
193 int error = 0;
194
195 if (drv->probe != NULL) {
196 error = driver_create_file(&drv->driver, &driver_attr_new_id);
197 if (error == 0) {
198 error = driver_create_file(&drv->driver,
199 &driver_attr_remove_id);
200 if (error)
201 driver_remove_file(&drv->driver,
202 &driver_attr_new_id);
203 }
204 }
205 return error;
206}
207
208static void pci_remove_newid_files(struct pci_driver *drv)
209{
210 driver_remove_file(&drv->driver, &driver_attr_remove_id);
211 driver_remove_file(&drv->driver, &driver_attr_new_id);
212}
213#else /* !CONFIG_HOTPLUG */
214static inline int pci_create_newid_files(struct pci_driver *drv)
215{
216 return 0;
217}
218static inline void pci_remove_newid_files(struct pci_driver *drv) {}
219#endif
220
221/**
222 * pci_match_id - See if a pci device matches a given pci_id table
223 * @ids: array of PCI device id structures to search in
224 * @dev: the PCI device structure to match against.
225 *
226 * Used by a driver to check whether a PCI device present in the
227 * system is in its list of supported devices. Returns the matching
228 * pci_device_id structure or %NULL if there is no match.
229 *
230 * Deprecated, don't use this as it will not catch any dynamic ids
231 * that a driver might want to check for.
232 */
233const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
234 struct pci_dev *dev)
235{
236 if (ids) {
237 while (ids->vendor || ids->subvendor || ids->class_mask) {
238 if (pci_match_one_device(ids, dev))
239 return ids;
240 ids++;
241 }
242 }
243 return NULL;
244}
245
246/**
247 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
248 * @drv: the PCI driver to match against
249 * @dev: the PCI device structure to match against
250 *
251 * Used by a driver to check whether a PCI device present in the
252 * system is in its list of supported devices. Returns the matching
253 * pci_device_id structure or %NULL if there is no match.
254 */
255static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
256 struct pci_dev *dev)
257{
258 struct pci_dynid *dynid;
259
260 /* Look at the dynamic ids first, before the static ones */
261 spin_lock(&drv->dynids.lock);
262 list_for_each_entry(dynid, &drv->dynids.list, node) {
263 if (pci_match_one_device(&dynid->id, dev)) {
264 spin_unlock(&drv->dynids.lock);
265 return &dynid->id;
266 }
267 }
268 spin_unlock(&drv->dynids.lock);
269
270 return pci_match_id(drv->id_table, dev);
271}
272
273struct drv_dev_and_id {
274 struct pci_driver *drv;
275 struct pci_dev *dev;
276 const struct pci_device_id *id;
277};
278
279static long local_pci_probe(void *_ddi)
280{
281 struct drv_dev_and_id *ddi = _ddi;
282 struct device *dev = &ddi->dev->dev;
283 int rc;
284
285 /* Unbound PCI devices are always set to disabled and suspended.
286 * During probe, the device is set to enabled and active and the
287 * usage count is incremented. If the driver supports runtime PM,
288 * it should call pm_runtime_put_noidle() in its probe routine and
289 * pm_runtime_get_noresume() in its remove routine.
290 */
291 pm_runtime_get_noresume(dev);
292 pm_runtime_set_active(dev);
293 pm_runtime_enable(dev);
294
295 rc = ddi->drv->probe(ddi->dev, ddi->id);
296 if (rc) {
297 pm_runtime_disable(dev);
298 pm_runtime_set_suspended(dev);
299 pm_runtime_put_noidle(dev);
300 }
301 return rc;
302}
303
304static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
305 const struct pci_device_id *id)
306{
307 int error, node;
308 struct drv_dev_and_id ddi = { drv, dev, id };
309
310 /* Execute driver initialization on node where the device's
311 bus is attached to. This way the driver likely allocates
312 its local memory on the right node without any need to
313 change it. */
314 node = dev_to_node(&dev->dev);
315 if (node >= 0) {
316 int cpu;
317
318 get_online_cpus();
319 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
320 if (cpu < nr_cpu_ids)
321 error = work_on_cpu(cpu, local_pci_probe, &ddi);
322 else
323 error = local_pci_probe(&ddi);
324 put_online_cpus();
325 } else
326 error = local_pci_probe(&ddi);
327 return error;
328}
329
330/**
331 * __pci_device_probe - check if a driver wants to claim a specific PCI device
332 * @drv: driver to call to check if it wants the PCI device
333 * @pci_dev: PCI device being probed
334 *
335 * returns 0 on success, else error.
336 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
337 */
338static int
339__pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
340{
341 const struct pci_device_id *id;
342 int error = 0;
343
344 if (!pci_dev->driver && drv->probe) {
345 error = -ENODEV;
346
347 id = pci_match_device(drv, pci_dev);
348 if (id)
349 error = pci_call_probe(drv, pci_dev, id);
350 if (error >= 0) {
351 pci_dev->driver = drv;
352 error = 0;
353 }
354 }
355 return error;
356}
357
358static int pci_device_probe(struct device * dev)
359{
360 int error = 0;
361 struct pci_driver *drv;
362 struct pci_dev *pci_dev;
363
364 drv = to_pci_driver(dev->driver);
365 pci_dev = to_pci_dev(dev);
366 pci_dev_get(pci_dev);
367 error = __pci_device_probe(drv, pci_dev);
368 if (error)
369 pci_dev_put(pci_dev);
370
371 return error;
372}
373
374static int pci_device_remove(struct device * dev)
375{
376 struct pci_dev * pci_dev = to_pci_dev(dev);
377 struct pci_driver * drv = pci_dev->driver;
378
379 if (drv) {
380 if (drv->remove) {
381 pm_runtime_get_sync(dev);
382 drv->remove(pci_dev);
383 pm_runtime_put_noidle(dev);
384 }
385 pci_dev->driver = NULL;
386 }
387
388 /* Undo the runtime PM settings in local_pci_probe() */
389 pm_runtime_disable(dev);
390 pm_runtime_set_suspended(dev);
391 pm_runtime_put_noidle(dev);
392
393 /*
394 * If the device is still on, set the power state as "unknown",
395 * since it might change by the next time we load the driver.
396 */
397 if (pci_dev->current_state == PCI_D0)
398 pci_dev->current_state = PCI_UNKNOWN;
399
400 /*
401 * We would love to complain here if pci_dev->is_enabled is set, that
402 * the driver should have called pci_disable_device(), but the
403 * unfortunate fact is there are too many odd BIOS and bridge setups
404 * that don't like drivers doing that all of the time.
405 * Oh well, we can dream of sane hardware when we sleep, no matter how
406 * horrible the crap we have to deal with is when we are awake...
407 */
408
409 pci_dev_put(pci_dev);
410 return 0;
411}
412
413static void pci_device_shutdown(struct device *dev)
414{
415 struct pci_dev *pci_dev = to_pci_dev(dev);
416 struct pci_driver *drv = pci_dev->driver;
417
418 if (drv && drv->shutdown)
419 drv->shutdown(pci_dev);
420 pci_msi_shutdown(pci_dev);
421 pci_msix_shutdown(pci_dev);
422
423 /*
424 * Turn off Bus Master bit on the device to tell it to not
425 * continue to do DMA
426 */
427 pci_disable_device(pci_dev);
428
429 /*
430 * Devices may be enabled to wake up by runtime PM, but they need not
431 * be supposed to wake up the system from its "power off" state (e.g.
432 * ACPI S5). Therefore disable wakeup for all devices that aren't
433 * supposed to wake up the system at this point. The state argument
434 * will be ignored by pci_enable_wake().
435 */
436 if (!device_may_wakeup(dev))
437 pci_enable_wake(pci_dev, PCI_UNKNOWN, false);
438}
439
440#ifdef CONFIG_PM
441
442/* Auxiliary functions used for system resume and run-time resume. */
443
444/**
445 * pci_restore_standard_config - restore standard config registers of PCI device
446 * @pci_dev: PCI device to handle
447 */
448static int pci_restore_standard_config(struct pci_dev *pci_dev)
449{
450 pci_update_current_state(pci_dev, PCI_UNKNOWN);
451
452 if (pci_dev->current_state != PCI_D0) {
453 int error = pci_set_power_state(pci_dev, PCI_D0);
454 if (error)
455 return error;
456 }
457
458 pci_restore_state(pci_dev);
459 return 0;
460}
461
462static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
463{
464 pci_restore_standard_config(pci_dev);
465 pci_fixup_device(pci_fixup_resume_early, pci_dev);
466}
467
468#endif
469
470#ifdef CONFIG_PM_SLEEP
471
472/*
473 * Default "suspend" method for devices that have no driver provided suspend,
474 * or not even a driver at all (second part).
475 */
476static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
477{
478 /*
479 * mark its power state as "unknown", since we don't know if
480 * e.g. the BIOS will change its device state when we suspend.
481 */
482 if (pci_dev->current_state == PCI_D0)
483 pci_dev->current_state = PCI_UNKNOWN;
484}
485
486/*
487 * Default "resume" method for devices that have no driver provided resume,
488 * or not even a driver at all (second part).
489 */
490static int pci_pm_reenable_device(struct pci_dev *pci_dev)
491{
492 int retval;
493
494 /* if the device was enabled before suspend, reenable */
495 retval = pci_reenable_device(pci_dev);
496 /*
497 * if the device was busmaster before the suspend, make it busmaster
498 * again
499 */
500 if (pci_dev->is_busmaster)
501 pci_set_master(pci_dev);
502
503 return retval;
504}
505
506static int pci_legacy_suspend(struct device *dev, pm_message_t state)
507{
508 struct pci_dev * pci_dev = to_pci_dev(dev);
509 struct pci_driver * drv = pci_dev->driver;
510
511 if (drv && drv->suspend) {
512 pci_power_t prev = pci_dev->current_state;
513 int error;
514
515 error = drv->suspend(pci_dev, state);
516 suspend_report_result(drv->suspend, error);
517 if (error)
518 return error;
519
520 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
521 && pci_dev->current_state != PCI_UNKNOWN) {
522 WARN_ONCE(pci_dev->current_state != prev,
523 "PCI PM: Device state not saved by %pF\n",
524 drv->suspend);
525 }
526 }
527
528 pci_fixup_device(pci_fixup_suspend, pci_dev);
529
530 return 0;
531}
532
533static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
534{
535 struct pci_dev * pci_dev = to_pci_dev(dev);
536 struct pci_driver * drv = pci_dev->driver;
537
538 if (drv && drv->suspend_late) {
539 pci_power_t prev = pci_dev->current_state;
540 int error;
541
542 error = drv->suspend_late(pci_dev, state);
543 suspend_report_result(drv->suspend_late, error);
544 if (error)
545 return error;
546
547 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
548 && pci_dev->current_state != PCI_UNKNOWN) {
549 WARN_ONCE(pci_dev->current_state != prev,
550 "PCI PM: Device state not saved by %pF\n",
551 drv->suspend_late);
552 return 0;
553 }
554 }
555
556 if (!pci_dev->state_saved)
557 pci_save_state(pci_dev);
558
559 pci_pm_set_unknown_state(pci_dev);
560
561 return 0;
562}
563
564static int pci_legacy_resume_early(struct device *dev)
565{
566 struct pci_dev * pci_dev = to_pci_dev(dev);
567 struct pci_driver * drv = pci_dev->driver;
568
569 return drv && drv->resume_early ?
570 drv->resume_early(pci_dev) : 0;
571}
572
573static int pci_legacy_resume(struct device *dev)
574{
575 struct pci_dev * pci_dev = to_pci_dev(dev);
576 struct pci_driver * drv = pci_dev->driver;
577
578 pci_fixup_device(pci_fixup_resume, pci_dev);
579
580 return drv && drv->resume ?
581 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
582}
583
584/* Auxiliary functions used by the new power management framework */
585
586static void pci_pm_default_resume(struct pci_dev *pci_dev)
587{
588 pci_fixup_device(pci_fixup_resume, pci_dev);
589
590 if (!pci_is_bridge(pci_dev))
591 pci_enable_wake(pci_dev, PCI_D0, false);
592}
593
594static void pci_pm_default_suspend(struct pci_dev *pci_dev)
595{
596 /* Disable non-bridge devices without PM support */
597 if (!pci_is_bridge(pci_dev))
598 pci_disable_enabled_device(pci_dev);
599}
600
601static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
602{
603 struct pci_driver *drv = pci_dev->driver;
604 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
605 || drv->resume_early);
606
607 /*
608 * Legacy PM support is used by default, so warn if the new framework is
609 * supported as well. Drivers are supposed to support either the
610 * former, or the latter, but not both at the same time.
611 */
612 WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
613 drv->name, pci_dev->vendor, pci_dev->device);
614
615 return ret;
616}
617
618/* New power management framework */
619
620static int pci_pm_prepare(struct device *dev)
621{
622 struct device_driver *drv = dev->driver;
623 int error = 0;
624
625 /*
626 * If a PCI device configured to wake up the system from sleep states
627 * has been suspended at run time and there's a resume request pending
628 * for it, this is equivalent to the device signaling wakeup, so the
629 * system suspend operation should be aborted.
630 */
631 pm_runtime_get_noresume(dev);
632 if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
633 pm_wakeup_event(dev, 0);
634
635 if (pm_wakeup_pending()) {
636 pm_runtime_put_sync(dev);
637 return -EBUSY;
638 }
639
640 /*
641 * PCI devices suspended at run time need to be resumed at this
642 * point, because in general it is necessary to reconfigure them for
643 * system suspend. Namely, if the device is supposed to wake up the
644 * system from the sleep state, we may need to reconfigure it for this
645 * purpose. In turn, if the device is not supposed to wake up the
646 * system from the sleep state, we'll have to prevent it from signaling
647 * wake-up.
648 */
649 pm_runtime_resume(dev);
650
651 if (drv && drv->pm && drv->pm->prepare)
652 error = drv->pm->prepare(dev);
653
654 return error;
655}
656
657static void pci_pm_complete(struct device *dev)
658{
659 struct device_driver *drv = dev->driver;
660
661 if (drv && drv->pm && drv->pm->complete)
662 drv->pm->complete(dev);
663
664 pm_runtime_put_sync(dev);
665}
666
667#else /* !CONFIG_PM_SLEEP */
668
669#define pci_pm_prepare NULL
670#define pci_pm_complete NULL
671
672#endif /* !CONFIG_PM_SLEEP */
673
674#ifdef CONFIG_SUSPEND
675
676static int pci_pm_suspend(struct device *dev)
677{
678 struct pci_dev *pci_dev = to_pci_dev(dev);
679 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
680
681 if (pci_has_legacy_pm_support(pci_dev))
682 return pci_legacy_suspend(dev, PMSG_SUSPEND);
683
684 if (!pm) {
685 pci_pm_default_suspend(pci_dev);
686 goto Fixup;
687 }
688
689 if (pm->suspend) {
690 pci_power_t prev = pci_dev->current_state;
691 int error;
692
693 error = pm->suspend(dev);
694 suspend_report_result(pm->suspend, error);
695 if (error)
696 return error;
697
698 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
699 && pci_dev->current_state != PCI_UNKNOWN) {
700 WARN_ONCE(pci_dev->current_state != prev,
701 "PCI PM: State of device not saved by %pF\n",
702 pm->suspend);
703 }
704 }
705
706 Fixup:
707 pci_fixup_device(pci_fixup_suspend, pci_dev);
708
709 return 0;
710}
711
712static int pci_pm_suspend_noirq(struct device *dev)
713{
714 struct pci_dev *pci_dev = to_pci_dev(dev);
715 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
716
717 if (pci_has_legacy_pm_support(pci_dev))
718 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
719
720 if (!pm) {
721 pci_save_state(pci_dev);
722 return 0;
723 }
724
725 if (pm->suspend_noirq) {
726 pci_power_t prev = pci_dev->current_state;
727 int error;
728
729 error = pm->suspend_noirq(dev);
730 suspend_report_result(pm->suspend_noirq, error);
731 if (error)
732 return error;
733
734 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
735 && pci_dev->current_state != PCI_UNKNOWN) {
736 WARN_ONCE(pci_dev->current_state != prev,
737 "PCI PM: State of device not saved by %pF\n",
738 pm->suspend_noirq);
739 return 0;
740 }
741 }
742
743 if (!pci_dev->state_saved) {
744 pci_save_state(pci_dev);
745 if (!pci_is_bridge(pci_dev))
746 pci_prepare_to_sleep(pci_dev);
747 }
748
749 pci_pm_set_unknown_state(pci_dev);
750
751 /*
752 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
753 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
754 * hasn't been quiesced and tries to turn it off. If the controller
755 * is already in D3, this can hang or cause memory corruption.
756 *
757 * Since the value of the COMMAND register doesn't matter once the
758 * device has been suspended, we can safely set it to 0 here.
759 */
760 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
761 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
762
763 return 0;
764}
765
766static int pci_pm_resume_noirq(struct device *dev)
767{
768 struct pci_dev *pci_dev = to_pci_dev(dev);
769 struct device_driver *drv = dev->driver;
770 int error = 0;
771
772 pci_pm_default_resume_early(pci_dev);
773
774 if (pci_has_legacy_pm_support(pci_dev))
775 return pci_legacy_resume_early(dev);
776
777 if (drv && drv->pm && drv->pm->resume_noirq)
778 error = drv->pm->resume_noirq(dev);
779
780 return error;
781}
782
783static int pci_pm_resume(struct device *dev)
784{
785 struct pci_dev *pci_dev = to_pci_dev(dev);
786 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
787 int error = 0;
788
789 /*
790 * This is necessary for the suspend error path in which resume is
791 * called without restoring the standard config registers of the device.
792 */
793 if (pci_dev->state_saved)
794 pci_restore_standard_config(pci_dev);
795
796 if (pci_has_legacy_pm_support(pci_dev))
797 return pci_legacy_resume(dev);
798
799 pci_pm_default_resume(pci_dev);
800
801 if (pm) {
802 if (pm->resume)
803 error = pm->resume(dev);
804 } else {
805 pci_pm_reenable_device(pci_dev);
806 }
807
808 return error;
809}
810
811#else /* !CONFIG_SUSPEND */
812
813#define pci_pm_suspend NULL
814#define pci_pm_suspend_noirq NULL
815#define pci_pm_resume NULL
816#define pci_pm_resume_noirq NULL
817
818#endif /* !CONFIG_SUSPEND */
819
820#ifdef CONFIG_HIBERNATE_CALLBACKS
821
822static int pci_pm_freeze(struct device *dev)
823{
824 struct pci_dev *pci_dev = to_pci_dev(dev);
825 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
826
827 if (pci_has_legacy_pm_support(pci_dev))
828 return pci_legacy_suspend(dev, PMSG_FREEZE);
829
830 if (!pm) {
831 pci_pm_default_suspend(pci_dev);
832 return 0;
833 }
834
835 if (pm->freeze) {
836 int error;
837
838 error = pm->freeze(dev);
839 suspend_report_result(pm->freeze, error);
840 if (error)
841 return error;
842 }
843
844 return 0;
845}
846
847static int pci_pm_freeze_noirq(struct device *dev)
848{
849 struct pci_dev *pci_dev = to_pci_dev(dev);
850 struct device_driver *drv = dev->driver;
851
852 if (pci_has_legacy_pm_support(pci_dev))
853 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
854
855 if (drv && drv->pm && drv->pm->freeze_noirq) {
856 int error;
857
858 error = drv->pm->freeze_noirq(dev);
859 suspend_report_result(drv->pm->freeze_noirq, error);
860 if (error)
861 return error;
862 }
863
864 if (!pci_dev->state_saved)
865 pci_save_state(pci_dev);
866
867 pci_pm_set_unknown_state(pci_dev);
868
869 return 0;
870}
871
872static int pci_pm_thaw_noirq(struct device *dev)
873{
874 struct pci_dev *pci_dev = to_pci_dev(dev);
875 struct device_driver *drv = dev->driver;
876 int error = 0;
877
878 if (pci_has_legacy_pm_support(pci_dev))
879 return pci_legacy_resume_early(dev);
880
881 pci_update_current_state(pci_dev, PCI_D0);
882
883 if (drv && drv->pm && drv->pm->thaw_noirq)
884 error = drv->pm->thaw_noirq(dev);
885
886 return error;
887}
888
889static int pci_pm_thaw(struct device *dev)
890{
891 struct pci_dev *pci_dev = to_pci_dev(dev);
892 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
893 int error = 0;
894
895 if (pci_has_legacy_pm_support(pci_dev))
896 return pci_legacy_resume(dev);
897
898 if (pm) {
899 if (pm->thaw)
900 error = pm->thaw(dev);
901 } else {
902 pci_pm_reenable_device(pci_dev);
903 }
904
905 pci_dev->state_saved = false;
906
907 return error;
908}
909
910static int pci_pm_poweroff(struct device *dev)
911{
912 struct pci_dev *pci_dev = to_pci_dev(dev);
913 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
914
915 if (pci_has_legacy_pm_support(pci_dev))
916 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
917
918 if (!pm) {
919 pci_pm_default_suspend(pci_dev);
920 goto Fixup;
921 }
922
923 if (pm->poweroff) {
924 int error;
925
926 error = pm->poweroff(dev);
927 suspend_report_result(pm->poweroff, error);
928 if (error)
929 return error;
930 }
931
932 Fixup:
933 pci_fixup_device(pci_fixup_suspend, pci_dev);
934
935 return 0;
936}
937
938static int pci_pm_poweroff_noirq(struct device *dev)
939{
940 struct pci_dev *pci_dev = to_pci_dev(dev);
941 struct device_driver *drv = dev->driver;
942
943 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
944 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
945
946 if (!drv || !drv->pm)
947 return 0;
948
949 if (drv->pm->poweroff_noirq) {
950 int error;
951
952 error = drv->pm->poweroff_noirq(dev);
953 suspend_report_result(drv->pm->poweroff_noirq, error);
954 if (error)
955 return error;
956 }
957
958 if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
959 pci_prepare_to_sleep(pci_dev);
960
961 /*
962 * The reason for doing this here is the same as for the analogous code
963 * in pci_pm_suspend_noirq().
964 */
965 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
966 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
967
968 return 0;
969}
970
971static int pci_pm_restore_noirq(struct device *dev)
972{
973 struct pci_dev *pci_dev = to_pci_dev(dev);
974 struct device_driver *drv = dev->driver;
975 int error = 0;
976
977 pci_pm_default_resume_early(pci_dev);
978
979 if (pci_has_legacy_pm_support(pci_dev))
980 return pci_legacy_resume_early(dev);
981
982 if (drv && drv->pm && drv->pm->restore_noirq)
983 error = drv->pm->restore_noirq(dev);
984
985 return error;
986}
987
988static int pci_pm_restore(struct device *dev)
989{
990 struct pci_dev *pci_dev = to_pci_dev(dev);
991 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
992 int error = 0;
993
994 /*
995 * This is necessary for the hibernation error path in which restore is
996 * called without restoring the standard config registers of the device.
997 */
998 if (pci_dev->state_saved)
999 pci_restore_standard_config(pci_dev);
1000
1001 if (pci_has_legacy_pm_support(pci_dev))
1002 return pci_legacy_resume(dev);
1003
1004 pci_pm_default_resume(pci_dev);
1005
1006 if (pm) {
1007 if (pm->restore)
1008 error = pm->restore(dev);
1009 } else {
1010 pci_pm_reenable_device(pci_dev);
1011 }
1012
1013 return error;
1014}
1015
1016#else /* !CONFIG_HIBERNATE_CALLBACKS */
1017
1018#define pci_pm_freeze NULL
1019#define pci_pm_freeze_noirq NULL
1020#define pci_pm_thaw NULL
1021#define pci_pm_thaw_noirq NULL
1022#define pci_pm_poweroff NULL
1023#define pci_pm_poweroff_noirq NULL
1024#define pci_pm_restore NULL
1025#define pci_pm_restore_noirq NULL
1026
1027#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1028
1029#ifdef CONFIG_PM_RUNTIME
1030
1031static int pci_pm_runtime_suspend(struct device *dev)
1032{
1033 struct pci_dev *pci_dev = to_pci_dev(dev);
1034 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1035 pci_power_t prev = pci_dev->current_state;
1036 int error;
1037
1038 if (!pm || !pm->runtime_suspend)
1039 return -ENOSYS;
1040
1041 error = pm->runtime_suspend(dev);
1042 suspend_report_result(pm->runtime_suspend, error);
1043 if (error)
1044 return error;
1045
1046 pci_fixup_device(pci_fixup_suspend, pci_dev);
1047
1048 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1049 && pci_dev->current_state != PCI_UNKNOWN) {
1050 WARN_ONCE(pci_dev->current_state != prev,
1051 "PCI PM: State of device not saved by %pF\n",
1052 pm->runtime_suspend);
1053 return 0;
1054 }
1055
1056 if (!pci_dev->state_saved)
1057 pci_save_state(pci_dev);
1058
1059 pci_finish_runtime_suspend(pci_dev);
1060
1061 return 0;
1062}
1063
1064static int pci_pm_runtime_resume(struct device *dev)
1065{
1066 struct pci_dev *pci_dev = to_pci_dev(dev);
1067 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1068
1069 if (!pm || !pm->runtime_resume)
1070 return -ENOSYS;
1071
1072 pci_pm_default_resume_early(pci_dev);
1073 __pci_enable_wake(pci_dev, PCI_D0, true, false);
1074 pci_fixup_device(pci_fixup_resume, pci_dev);
1075
1076 return pm->runtime_resume(dev);
1077}
1078
1079static int pci_pm_runtime_idle(struct device *dev)
1080{
1081 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1082
1083 if (!pm)
1084 return -ENOSYS;
1085
1086 if (pm->runtime_idle) {
1087 int ret = pm->runtime_idle(dev);
1088 if (ret)
1089 return ret;
1090 }
1091
1092 pm_runtime_suspend(dev);
1093
1094 return 0;
1095}
1096
1097#else /* !CONFIG_PM_RUNTIME */
1098
1099#define pci_pm_runtime_suspend NULL
1100#define pci_pm_runtime_resume NULL
1101#define pci_pm_runtime_idle NULL
1102
1103#endif /* !CONFIG_PM_RUNTIME */
1104
1105#ifdef CONFIG_PM
1106
1107const struct dev_pm_ops pci_dev_pm_ops = {
1108 .prepare = pci_pm_prepare,
1109 .complete = pci_pm_complete,
1110 .suspend = pci_pm_suspend,
1111 .resume = pci_pm_resume,
1112 .freeze = pci_pm_freeze,
1113 .thaw = pci_pm_thaw,
1114 .poweroff = pci_pm_poweroff,
1115 .restore = pci_pm_restore,
1116 .suspend_noirq = pci_pm_suspend_noirq,
1117 .resume_noirq = pci_pm_resume_noirq,
1118 .freeze_noirq = pci_pm_freeze_noirq,
1119 .thaw_noirq = pci_pm_thaw_noirq,
1120 .poweroff_noirq = pci_pm_poweroff_noirq,
1121 .restore_noirq = pci_pm_restore_noirq,
1122 .runtime_suspend = pci_pm_runtime_suspend,
1123 .runtime_resume = pci_pm_runtime_resume,
1124 .runtime_idle = pci_pm_runtime_idle,
1125};
1126
1127#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1128
1129#else /* !COMFIG_PM_OPS */
1130
1131#define PCI_PM_OPS_PTR NULL
1132
1133#endif /* !COMFIG_PM_OPS */
1134
1135/**
1136 * __pci_register_driver - register a new pci driver
1137 * @drv: the driver structure to register
1138 * @owner: owner module of drv
1139 * @mod_name: module name string
1140 *
1141 * Adds the driver structure to the list of registered drivers.
1142 * Returns a negative value on error, otherwise 0.
1143 * If no error occurred, the driver remains registered even if
1144 * no device was claimed during registration.
1145 */
1146int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1147 const char *mod_name)
1148{
1149 int error;
1150
1151 /* initialize common driver fields */
1152 drv->driver.name = drv->name;
1153 drv->driver.bus = &pci_bus_type;
1154 drv->driver.owner = owner;
1155 drv->driver.mod_name = mod_name;
1156
1157 spin_lock_init(&drv->dynids.lock);
1158 INIT_LIST_HEAD(&drv->dynids.list);
1159
1160 /* register with core */
1161 error = driver_register(&drv->driver);
1162 if (error)
1163 goto out;
1164
1165 error = pci_create_newid_files(drv);
1166 if (error)
1167 goto out_newid;
1168out:
1169 return error;
1170
1171out_newid:
1172 driver_unregister(&drv->driver);
1173 goto out;
1174}
1175
1176/**
1177 * pci_unregister_driver - unregister a pci driver
1178 * @drv: the driver structure to unregister
1179 *
1180 * Deletes the driver structure from the list of registered PCI drivers,
1181 * gives it a chance to clean up by calling its remove() function for
1182 * each device it was responsible for, and marks those devices as
1183 * driverless.
1184 */
1185
1186void
1187pci_unregister_driver(struct pci_driver *drv)
1188{
1189 pci_remove_newid_files(drv);
1190 driver_unregister(&drv->driver);
1191 pci_free_dynids(drv);
1192}
1193
1194static struct pci_driver pci_compat_driver = {
1195 .name = "compat"
1196};
1197
1198/**
1199 * pci_dev_driver - get the pci_driver of a device
1200 * @dev: the device to query
1201 *
1202 * Returns the appropriate pci_driver structure or %NULL if there is no
1203 * registered driver for the device.
1204 */
1205struct pci_driver *
1206pci_dev_driver(const struct pci_dev *dev)
1207{
1208 if (dev->driver)
1209 return dev->driver;
1210 else {
1211 int i;
1212 for(i=0; i<=PCI_ROM_RESOURCE; i++)
1213 if (dev->resource[i].flags & IORESOURCE_BUSY)
1214 return &pci_compat_driver;
1215 }
1216 return NULL;
1217}
1218
1219/**
1220 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1221 * @dev: the PCI device structure to match against
1222 * @drv: the device driver to search for matching PCI device id structures
1223 *
1224 * Used by a driver to check whether a PCI device present in the
1225 * system is in its list of supported devices. Returns the matching
1226 * pci_device_id structure or %NULL if there is no match.
1227 */
1228static int pci_bus_match(struct device *dev, struct device_driver *drv)
1229{
1230 struct pci_dev *pci_dev = to_pci_dev(dev);
1231 struct pci_driver *pci_drv = to_pci_driver(drv);
1232 const struct pci_device_id *found_id;
1233
1234 found_id = pci_match_device(pci_drv, pci_dev);
1235 if (found_id)
1236 return 1;
1237
1238 return 0;
1239}
1240
1241/**
1242 * pci_dev_get - increments the reference count of the pci device structure
1243 * @dev: the device being referenced
1244 *
1245 * Each live reference to a device should be refcounted.
1246 *
1247 * Drivers for PCI devices should normally record such references in
1248 * their probe() methods, when they bind to a device, and release
1249 * them by calling pci_dev_put(), in their disconnect() methods.
1250 *
1251 * A pointer to the device with the incremented reference counter is returned.
1252 */
1253struct pci_dev *pci_dev_get(struct pci_dev *dev)
1254{
1255 if (dev)
1256 get_device(&dev->dev);
1257 return dev;
1258}
1259
1260/**
1261 * pci_dev_put - release a use of the pci device structure
1262 * @dev: device that's been disconnected
1263 *
1264 * Must be called when a user of a device is finished with it. When the last
1265 * user of the device calls this function, the memory of the device is freed.
1266 */
1267void pci_dev_put(struct pci_dev *dev)
1268{
1269 if (dev)
1270 put_device(&dev->dev);
1271}
1272
1273#ifndef CONFIG_HOTPLUG
1274int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1275{
1276 return -ENODEV;
1277}
1278#endif
1279
1280struct bus_type pci_bus_type = {
1281 .name = "pci",
1282 .match = pci_bus_match,
1283 .uevent = pci_uevent,
1284 .probe = pci_device_probe,
1285 .remove = pci_device_remove,
1286 .shutdown = pci_device_shutdown,
1287 .dev_attrs = pci_dev_attrs,
1288 .bus_attrs = pci_bus_attrs,
1289 .pm = PCI_PM_OPS_PTR,
1290};
1291
1292static int __init pci_driver_init(void)
1293{
1294 return bus_register(&pci_bus_type);
1295}
1296
1297postcore_initcall(pci_driver_init);
1298
1299EXPORT_SYMBOL_GPL(pci_add_dynid);
1300EXPORT_SYMBOL(pci_match_id);
1301EXPORT_SYMBOL(__pci_register_driver);
1302EXPORT_SYMBOL(pci_unregister_driver);
1303EXPORT_SYMBOL(pci_dev_driver);
1304EXPORT_SYMBOL(pci_bus_type);
1305EXPORT_SYMBOL(pci_dev_get);
1306EXPORT_SYMBOL(pci_dev_put);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
4 * (C) Copyright 2007 Novell Inc.
5 */
6
7#include <linux/pci.h>
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/device.h>
11#include <linux/mempolicy.h>
12#include <linux/string.h>
13#include <linux/slab.h>
14#include <linux/sched.h>
15#include <linux/sched/isolation.h>
16#include <linux/cpu.h>
17#include <linux/pm_runtime.h>
18#include <linux/suspend.h>
19#include <linux/kexec.h>
20#include <linux/of_device.h>
21#include <linux/acpi.h>
22#include <linux/dma-map-ops.h>
23#include <linux/iommu.h>
24#include "pci.h"
25#include "pcie/portdrv.h"
26
27struct pci_dynid {
28 struct list_head node;
29 struct pci_device_id id;
30};
31
32/**
33 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
34 * @drv: target pci driver
35 * @vendor: PCI vendor ID
36 * @device: PCI device ID
37 * @subvendor: PCI subvendor ID
38 * @subdevice: PCI subdevice ID
39 * @class: PCI class
40 * @class_mask: PCI class mask
41 * @driver_data: private driver data
42 *
43 * Adds a new dynamic pci device ID to this driver and causes the
44 * driver to probe for all devices again. @drv must have been
45 * registered prior to calling this function.
46 *
47 * CONTEXT:
48 * Does GFP_KERNEL allocation.
49 *
50 * RETURNS:
51 * 0 on success, -errno on failure.
52 */
53int pci_add_dynid(struct pci_driver *drv,
54 unsigned int vendor, unsigned int device,
55 unsigned int subvendor, unsigned int subdevice,
56 unsigned int class, unsigned int class_mask,
57 unsigned long driver_data)
58{
59 struct pci_dynid *dynid;
60
61 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
62 if (!dynid)
63 return -ENOMEM;
64
65 dynid->id.vendor = vendor;
66 dynid->id.device = device;
67 dynid->id.subvendor = subvendor;
68 dynid->id.subdevice = subdevice;
69 dynid->id.class = class;
70 dynid->id.class_mask = class_mask;
71 dynid->id.driver_data = driver_data;
72
73 spin_lock(&drv->dynids.lock);
74 list_add_tail(&dynid->node, &drv->dynids.list);
75 spin_unlock(&drv->dynids.lock);
76
77 return driver_attach(&drv->driver);
78}
79EXPORT_SYMBOL_GPL(pci_add_dynid);
80
81static void pci_free_dynids(struct pci_driver *drv)
82{
83 struct pci_dynid *dynid, *n;
84
85 spin_lock(&drv->dynids.lock);
86 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
87 list_del(&dynid->node);
88 kfree(dynid);
89 }
90 spin_unlock(&drv->dynids.lock);
91}
92
93/**
94 * pci_match_id - See if a PCI device matches a given pci_id table
95 * @ids: array of PCI device ID structures to search in
96 * @dev: the PCI device structure to match against.
97 *
98 * Used by a driver to check whether a PCI device is in its list of
99 * supported devices. Returns the matching pci_device_id structure or
100 * %NULL if there is no match.
101 *
102 * Deprecated; don't use this as it will not catch any dynamic IDs
103 * that a driver might want to check for.
104 */
105const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
106 struct pci_dev *dev)
107{
108 if (ids) {
109 while (ids->vendor || ids->subvendor || ids->class_mask) {
110 if (pci_match_one_device(ids, dev))
111 return ids;
112 ids++;
113 }
114 }
115 return NULL;
116}
117EXPORT_SYMBOL(pci_match_id);
118
119static const struct pci_device_id pci_device_id_any = {
120 .vendor = PCI_ANY_ID,
121 .device = PCI_ANY_ID,
122 .subvendor = PCI_ANY_ID,
123 .subdevice = PCI_ANY_ID,
124};
125
126/**
127 * pci_match_device - See if a device matches a driver's list of IDs
128 * @drv: the PCI driver to match against
129 * @dev: the PCI device structure to match against
130 *
131 * Used by a driver to check whether a PCI device is in its list of
132 * supported devices or in the dynids list, which may have been augmented
133 * via the sysfs "new_id" file. Returns the matching pci_device_id
134 * structure or %NULL if there is no match.
135 */
136static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
137 struct pci_dev *dev)
138{
139 struct pci_dynid *dynid;
140 const struct pci_device_id *found_id = NULL, *ids;
141
142 /* When driver_override is set, only bind to the matching driver */
143 if (dev->driver_override && strcmp(dev->driver_override, drv->name))
144 return NULL;
145
146 /* Look at the dynamic ids first, before the static ones */
147 spin_lock(&drv->dynids.lock);
148 list_for_each_entry(dynid, &drv->dynids.list, node) {
149 if (pci_match_one_device(&dynid->id, dev)) {
150 found_id = &dynid->id;
151 break;
152 }
153 }
154 spin_unlock(&drv->dynids.lock);
155
156 if (found_id)
157 return found_id;
158
159 for (ids = drv->id_table; (found_id = pci_match_id(ids, dev));
160 ids = found_id + 1) {
161 /*
162 * The match table is split based on driver_override.
163 * In case override_only was set, enforce driver_override
164 * matching.
165 */
166 if (found_id->override_only) {
167 if (dev->driver_override)
168 return found_id;
169 } else {
170 return found_id;
171 }
172 }
173
174 /* driver_override will always match, send a dummy id */
175 if (dev->driver_override)
176 return &pci_device_id_any;
177 return NULL;
178}
179
180/**
181 * new_id_store - sysfs frontend to pci_add_dynid()
182 * @driver: target device driver
183 * @buf: buffer for scanning device ID data
184 * @count: input size
185 *
186 * Allow PCI IDs to be added to an existing driver via sysfs.
187 */
188static ssize_t new_id_store(struct device_driver *driver, const char *buf,
189 size_t count)
190{
191 struct pci_driver *pdrv = to_pci_driver(driver);
192 const struct pci_device_id *ids = pdrv->id_table;
193 u32 vendor, device, subvendor = PCI_ANY_ID,
194 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
195 unsigned long driver_data = 0;
196 int fields;
197 int retval = 0;
198
199 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
200 &vendor, &device, &subvendor, &subdevice,
201 &class, &class_mask, &driver_data);
202 if (fields < 2)
203 return -EINVAL;
204
205 if (fields != 7) {
206 struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
207 if (!pdev)
208 return -ENOMEM;
209
210 pdev->vendor = vendor;
211 pdev->device = device;
212 pdev->subsystem_vendor = subvendor;
213 pdev->subsystem_device = subdevice;
214 pdev->class = class;
215
216 if (pci_match_device(pdrv, pdev))
217 retval = -EEXIST;
218
219 kfree(pdev);
220
221 if (retval)
222 return retval;
223 }
224
225 /* Only accept driver_data values that match an existing id_table
226 entry */
227 if (ids) {
228 retval = -EINVAL;
229 while (ids->vendor || ids->subvendor || ids->class_mask) {
230 if (driver_data == ids->driver_data) {
231 retval = 0;
232 break;
233 }
234 ids++;
235 }
236 if (retval) /* No match */
237 return retval;
238 }
239
240 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
241 class, class_mask, driver_data);
242 if (retval)
243 return retval;
244 return count;
245}
246static DRIVER_ATTR_WO(new_id);
247
248/**
249 * remove_id_store - remove a PCI device ID from this driver
250 * @driver: target device driver
251 * @buf: buffer for scanning device ID data
252 * @count: input size
253 *
254 * Removes a dynamic pci device ID to this driver.
255 */
256static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
257 size_t count)
258{
259 struct pci_dynid *dynid, *n;
260 struct pci_driver *pdrv = to_pci_driver(driver);
261 u32 vendor, device, subvendor = PCI_ANY_ID,
262 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
263 int fields;
264 size_t retval = -ENODEV;
265
266 fields = sscanf(buf, "%x %x %x %x %x %x",
267 &vendor, &device, &subvendor, &subdevice,
268 &class, &class_mask);
269 if (fields < 2)
270 return -EINVAL;
271
272 spin_lock(&pdrv->dynids.lock);
273 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
274 struct pci_device_id *id = &dynid->id;
275 if ((id->vendor == vendor) &&
276 (id->device == device) &&
277 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
278 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
279 !((id->class ^ class) & class_mask)) {
280 list_del(&dynid->node);
281 kfree(dynid);
282 retval = count;
283 break;
284 }
285 }
286 spin_unlock(&pdrv->dynids.lock);
287
288 return retval;
289}
290static DRIVER_ATTR_WO(remove_id);
291
292static struct attribute *pci_drv_attrs[] = {
293 &driver_attr_new_id.attr,
294 &driver_attr_remove_id.attr,
295 NULL,
296};
297ATTRIBUTE_GROUPS(pci_drv);
298
299struct drv_dev_and_id {
300 struct pci_driver *drv;
301 struct pci_dev *dev;
302 const struct pci_device_id *id;
303};
304
305static long local_pci_probe(void *_ddi)
306{
307 struct drv_dev_and_id *ddi = _ddi;
308 struct pci_dev *pci_dev = ddi->dev;
309 struct pci_driver *pci_drv = ddi->drv;
310 struct device *dev = &pci_dev->dev;
311 int rc;
312
313 /*
314 * Unbound PCI devices are always put in D0, regardless of
315 * runtime PM status. During probe, the device is set to
316 * active and the usage count is incremented. If the driver
317 * supports runtime PM, it should call pm_runtime_put_noidle(),
318 * or any other runtime PM helper function decrementing the usage
319 * count, in its probe routine and pm_runtime_get_noresume() in
320 * its remove routine.
321 */
322 pm_runtime_get_sync(dev);
323 pci_dev->driver = pci_drv;
324 rc = pci_drv->probe(pci_dev, ddi->id);
325 if (!rc)
326 return rc;
327 if (rc < 0) {
328 pci_dev->driver = NULL;
329 pm_runtime_put_sync(dev);
330 return rc;
331 }
332 /*
333 * Probe function should return < 0 for failure, 0 for success
334 * Treat values > 0 as success, but warn.
335 */
336 pci_warn(pci_dev, "Driver probe function unexpectedly returned %d\n",
337 rc);
338 return 0;
339}
340
341static bool pci_physfn_is_probed(struct pci_dev *dev)
342{
343#ifdef CONFIG_PCI_IOV
344 return dev->is_virtfn && dev->physfn->is_probed;
345#else
346 return false;
347#endif
348}
349
350static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
351 const struct pci_device_id *id)
352{
353 int error, node, cpu;
354 struct drv_dev_and_id ddi = { drv, dev, id };
355
356 /*
357 * Execute driver initialization on node where the device is
358 * attached. This way the driver likely allocates its local memory
359 * on the right node.
360 */
361 node = dev_to_node(&dev->dev);
362 dev->is_probed = 1;
363
364 cpu_hotplug_disable();
365
366 /*
367 * Prevent nesting work_on_cpu() for the case where a Virtual Function
368 * device is probed from work_on_cpu() of the Physical device.
369 */
370 if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
371 pci_physfn_is_probed(dev)) {
372 cpu = nr_cpu_ids;
373 } else {
374 cpumask_var_t wq_domain_mask;
375
376 if (!zalloc_cpumask_var(&wq_domain_mask, GFP_KERNEL)) {
377 error = -ENOMEM;
378 goto out;
379 }
380 cpumask_and(wq_domain_mask,
381 housekeeping_cpumask(HK_TYPE_WQ),
382 housekeeping_cpumask(HK_TYPE_DOMAIN));
383
384 cpu = cpumask_any_and(cpumask_of_node(node),
385 wq_domain_mask);
386 free_cpumask_var(wq_domain_mask);
387 }
388
389 if (cpu < nr_cpu_ids)
390 error = work_on_cpu(cpu, local_pci_probe, &ddi);
391 else
392 error = local_pci_probe(&ddi);
393out:
394 dev->is_probed = 0;
395 cpu_hotplug_enable();
396 return error;
397}
398
399/**
400 * __pci_device_probe - check if a driver wants to claim a specific PCI device
401 * @drv: driver to call to check if it wants the PCI device
402 * @pci_dev: PCI device being probed
403 *
404 * returns 0 on success, else error.
405 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
406 */
407static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
408{
409 const struct pci_device_id *id;
410 int error = 0;
411
412 if (drv->probe) {
413 error = -ENODEV;
414
415 id = pci_match_device(drv, pci_dev);
416 if (id)
417 error = pci_call_probe(drv, pci_dev, id);
418 }
419 return error;
420}
421
422#ifdef CONFIG_PCI_IOV
423static inline bool pci_device_can_probe(struct pci_dev *pdev)
424{
425 return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe ||
426 pdev->driver_override);
427}
428#else
429static inline bool pci_device_can_probe(struct pci_dev *pdev)
430{
431 return true;
432}
433#endif
434
435static int pci_device_probe(struct device *dev)
436{
437 int error;
438 struct pci_dev *pci_dev = to_pci_dev(dev);
439 struct pci_driver *drv = to_pci_driver(dev->driver);
440
441 if (!pci_device_can_probe(pci_dev))
442 return -ENODEV;
443
444 pci_assign_irq(pci_dev);
445
446 error = pcibios_alloc_irq(pci_dev);
447 if (error < 0)
448 return error;
449
450 pci_dev_get(pci_dev);
451 error = __pci_device_probe(drv, pci_dev);
452 if (error) {
453 pcibios_free_irq(pci_dev);
454 pci_dev_put(pci_dev);
455 }
456
457 return error;
458}
459
460static void pci_device_remove(struct device *dev)
461{
462 struct pci_dev *pci_dev = to_pci_dev(dev);
463 struct pci_driver *drv = pci_dev->driver;
464
465 if (drv->remove) {
466 pm_runtime_get_sync(dev);
467 /*
468 * If the driver provides a .runtime_idle() callback and it has
469 * started to run already, it may continue to run in parallel
470 * with the code below, so wait until all of the runtime PM
471 * activity has completed.
472 */
473 pm_runtime_barrier(dev);
474 drv->remove(pci_dev);
475 pm_runtime_put_noidle(dev);
476 }
477 pcibios_free_irq(pci_dev);
478 pci_dev->driver = NULL;
479 pci_iov_remove(pci_dev);
480
481 /* Undo the runtime PM settings in local_pci_probe() */
482 pm_runtime_put_sync(dev);
483
484 /*
485 * If the device is still on, set the power state as "unknown",
486 * since it might change by the next time we load the driver.
487 */
488 if (pci_dev->current_state == PCI_D0)
489 pci_dev->current_state = PCI_UNKNOWN;
490
491 /*
492 * We would love to complain here if pci_dev->is_enabled is set, that
493 * the driver should have called pci_disable_device(), but the
494 * unfortunate fact is there are too many odd BIOS and bridge setups
495 * that don't like drivers doing that all of the time.
496 * Oh well, we can dream of sane hardware when we sleep, no matter how
497 * horrible the crap we have to deal with is when we are awake...
498 */
499
500 pci_dev_put(pci_dev);
501}
502
503static void pci_device_shutdown(struct device *dev)
504{
505 struct pci_dev *pci_dev = to_pci_dev(dev);
506 struct pci_driver *drv = pci_dev->driver;
507
508 pm_runtime_resume(dev);
509
510 if (drv && drv->shutdown)
511 drv->shutdown(pci_dev);
512
513 /*
514 * If this is a kexec reboot, turn off Bus Master bit on the
515 * device to tell it to not continue to do DMA. Don't touch
516 * devices in D3cold or unknown states.
517 * If it is not a kexec reboot, firmware will hit the PCI
518 * devices with big hammer and stop their DMA any way.
519 */
520 if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
521 pci_clear_master(pci_dev);
522}
523
524#ifdef CONFIG_PM_SLEEP
525
526/* Auxiliary functions used for system resume */
527
528/**
529 * pci_restore_standard_config - restore standard config registers of PCI device
530 * @pci_dev: PCI device to handle
531 */
532static int pci_restore_standard_config(struct pci_dev *pci_dev)
533{
534 pci_update_current_state(pci_dev, PCI_UNKNOWN);
535
536 if (pci_dev->current_state != PCI_D0) {
537 int error = pci_set_power_state(pci_dev, PCI_D0);
538 if (error)
539 return error;
540 }
541
542 pci_restore_state(pci_dev);
543 pci_pme_restore(pci_dev);
544 return 0;
545}
546#endif /* CONFIG_PM_SLEEP */
547
548#ifdef CONFIG_PM
549
550/* Auxiliary functions used for system resume and run-time resume */
551
552static void pci_pm_default_resume(struct pci_dev *pci_dev)
553{
554 pci_fixup_device(pci_fixup_resume, pci_dev);
555 pci_enable_wake(pci_dev, PCI_D0, false);
556}
557
558static void pci_pm_power_up_and_verify_state(struct pci_dev *pci_dev)
559{
560 pci_power_up(pci_dev);
561 pci_update_current_state(pci_dev, PCI_D0);
562}
563
564static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
565{
566 pci_pm_power_up_and_verify_state(pci_dev);
567 pci_restore_state(pci_dev);
568 pci_pme_restore(pci_dev);
569}
570
571static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev)
572{
573 int ret;
574
575 ret = pci_bridge_wait_for_secondary_bus(pci_dev, "resume");
576 if (ret) {
577 /*
578 * The downstream link failed to come up, so mark the
579 * devices below as disconnected to make sure we don't
580 * attempt to resume them.
581 */
582 pci_walk_bus(pci_dev->subordinate, pci_dev_set_disconnected,
583 NULL);
584 return;
585 }
586
587 /*
588 * When powering on a bridge from D3cold, the whole hierarchy may be
589 * powered on into D0uninitialized state, resume them to give them a
590 * chance to suspend again
591 */
592 pci_resume_bus(pci_dev->subordinate);
593}
594
595#endif /* CONFIG_PM */
596
597#ifdef CONFIG_PM_SLEEP
598
599/*
600 * Default "suspend" method for devices that have no driver provided suspend,
601 * or not even a driver at all (second part).
602 */
603static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
604{
605 /*
606 * mark its power state as "unknown", since we don't know if
607 * e.g. the BIOS will change its device state when we suspend.
608 */
609 if (pci_dev->current_state == PCI_D0)
610 pci_dev->current_state = PCI_UNKNOWN;
611}
612
613/*
614 * Default "resume" method for devices that have no driver provided resume,
615 * or not even a driver at all (second part).
616 */
617static int pci_pm_reenable_device(struct pci_dev *pci_dev)
618{
619 int retval;
620
621 /* if the device was enabled before suspend, re-enable */
622 retval = pci_reenable_device(pci_dev);
623 /*
624 * if the device was busmaster before the suspend, make it busmaster
625 * again
626 */
627 if (pci_dev->is_busmaster)
628 pci_set_master(pci_dev);
629
630 return retval;
631}
632
633static int pci_legacy_suspend(struct device *dev, pm_message_t state)
634{
635 struct pci_dev *pci_dev = to_pci_dev(dev);
636 struct pci_driver *drv = pci_dev->driver;
637
638 if (drv && drv->suspend) {
639 pci_power_t prev = pci_dev->current_state;
640 int error;
641
642 error = drv->suspend(pci_dev, state);
643 suspend_report_result(dev, drv->suspend, error);
644 if (error)
645 return error;
646
647 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
648 && pci_dev->current_state != PCI_UNKNOWN) {
649 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
650 "PCI PM: Device state not saved by %pS\n",
651 drv->suspend);
652 }
653 }
654
655 pci_fixup_device(pci_fixup_suspend, pci_dev);
656
657 return 0;
658}
659
660static int pci_legacy_suspend_late(struct device *dev)
661{
662 struct pci_dev *pci_dev = to_pci_dev(dev);
663
664 if (!pci_dev->state_saved)
665 pci_save_state(pci_dev);
666
667 pci_pm_set_unknown_state(pci_dev);
668
669 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
670
671 return 0;
672}
673
674static int pci_legacy_resume(struct device *dev)
675{
676 struct pci_dev *pci_dev = to_pci_dev(dev);
677 struct pci_driver *drv = pci_dev->driver;
678
679 pci_fixup_device(pci_fixup_resume, pci_dev);
680
681 return drv && drv->resume ?
682 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
683}
684
685/* Auxiliary functions used by the new power management framework */
686
687static void pci_pm_default_suspend(struct pci_dev *pci_dev)
688{
689 /* Disable non-bridge devices without PM support */
690 if (!pci_has_subordinate(pci_dev))
691 pci_disable_enabled_device(pci_dev);
692}
693
694static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
695{
696 struct pci_driver *drv = pci_dev->driver;
697 bool ret = drv && (drv->suspend || drv->resume);
698
699 /*
700 * Legacy PM support is used by default, so warn if the new framework is
701 * supported as well. Drivers are supposed to support either the
702 * former, or the latter, but not both at the same time.
703 */
704 pci_WARN(pci_dev, ret && drv->driver.pm, "device %04x:%04x\n",
705 pci_dev->vendor, pci_dev->device);
706
707 return ret;
708}
709
710/* New power management framework */
711
712static int pci_pm_prepare(struct device *dev)
713{
714 struct pci_dev *pci_dev = to_pci_dev(dev);
715 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
716
717 if (pm && pm->prepare) {
718 int error = pm->prepare(dev);
719 if (error < 0)
720 return error;
721
722 if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
723 return 0;
724 }
725 if (pci_dev_need_resume(pci_dev))
726 return 0;
727
728 /*
729 * The PME setting needs to be adjusted here in case the direct-complete
730 * optimization is used with respect to this device.
731 */
732 pci_dev_adjust_pme(pci_dev);
733 return 1;
734}
735
736static void pci_pm_complete(struct device *dev)
737{
738 struct pci_dev *pci_dev = to_pci_dev(dev);
739
740 pci_dev_complete_resume(pci_dev);
741 pm_generic_complete(dev);
742
743 /* Resume device if platform firmware has put it in reset-power-on */
744 if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
745 pci_power_t pre_sleep_state = pci_dev->current_state;
746
747 pci_refresh_power_state(pci_dev);
748 /*
749 * On platforms with ACPI this check may also trigger for
750 * devices sharing power resources if one of those power
751 * resources has been activated as a result of a change of the
752 * power state of another device sharing it. However, in that
753 * case it is also better to resume the device, in general.
754 */
755 if (pci_dev->current_state < pre_sleep_state)
756 pm_request_resume(dev);
757 }
758}
759
760#else /* !CONFIG_PM_SLEEP */
761
762#define pci_pm_prepare NULL
763#define pci_pm_complete NULL
764
765#endif /* !CONFIG_PM_SLEEP */
766
767#ifdef CONFIG_SUSPEND
768static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
769{
770 /*
771 * Some BIOSes forget to clear Root PME Status bits after system
772 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
773 * Clear those bits now just in case (shouldn't hurt).
774 */
775 if (pci_is_pcie(pci_dev) &&
776 (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
777 pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
778 pcie_clear_root_pme_status(pci_dev);
779}
780
781static int pci_pm_suspend(struct device *dev)
782{
783 struct pci_dev *pci_dev = to_pci_dev(dev);
784 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
785
786 pci_dev->skip_bus_pm = false;
787
788 /*
789 * Disabling PTM allows some systems, e.g., Intel mobile chips
790 * since Coffee Lake, to enter a lower-power PM state.
791 */
792 pci_suspend_ptm(pci_dev);
793
794 if (pci_has_legacy_pm_support(pci_dev))
795 return pci_legacy_suspend(dev, PMSG_SUSPEND);
796
797 if (!pm) {
798 pci_pm_default_suspend(pci_dev);
799 return 0;
800 }
801
802 /*
803 * PCI devices suspended at run time may need to be resumed at this
804 * point, because in general it may be necessary to reconfigure them for
805 * system suspend. Namely, if the device is expected to wake up the
806 * system from the sleep state, it may have to be reconfigured for this
807 * purpose, or if the device is not expected to wake up the system from
808 * the sleep state, it should be prevented from signaling wakeup events
809 * going forward.
810 *
811 * Also if the driver of the device does not indicate that its system
812 * suspend callbacks can cope with runtime-suspended devices, it is
813 * better to resume the device from runtime suspend here.
814 */
815 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
816 pci_dev_need_resume(pci_dev)) {
817 pm_runtime_resume(dev);
818 pci_dev->state_saved = false;
819 } else {
820 pci_dev_adjust_pme(pci_dev);
821 }
822
823 if (pm->suspend) {
824 pci_power_t prev = pci_dev->current_state;
825 int error;
826
827 error = pm->suspend(dev);
828 suspend_report_result(dev, pm->suspend, error);
829 if (error)
830 return error;
831
832 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
833 && pci_dev->current_state != PCI_UNKNOWN) {
834 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
835 "PCI PM: State of device not saved by %pS\n",
836 pm->suspend);
837 }
838 }
839
840 return 0;
841}
842
843static int pci_pm_suspend_late(struct device *dev)
844{
845 if (dev_pm_skip_suspend(dev))
846 return 0;
847
848 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
849
850 return pm_generic_suspend_late(dev);
851}
852
853static int pci_pm_suspend_noirq(struct device *dev)
854{
855 struct pci_dev *pci_dev = to_pci_dev(dev);
856 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
857
858 if (dev_pm_skip_suspend(dev))
859 return 0;
860
861 if (pci_has_legacy_pm_support(pci_dev))
862 return pci_legacy_suspend_late(dev);
863
864 if (!pm) {
865 pci_save_state(pci_dev);
866 goto Fixup;
867 }
868
869 if (pm->suspend_noirq) {
870 pci_power_t prev = pci_dev->current_state;
871 int error;
872
873 error = pm->suspend_noirq(dev);
874 suspend_report_result(dev, pm->suspend_noirq, error);
875 if (error)
876 return error;
877
878 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
879 && pci_dev->current_state != PCI_UNKNOWN) {
880 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
881 "PCI PM: State of device not saved by %pS\n",
882 pm->suspend_noirq);
883 goto Fixup;
884 }
885 }
886
887 if (!pci_dev->state_saved) {
888 pci_save_state(pci_dev);
889
890 /*
891 * If the device is a bridge with a child in D0 below it,
892 * it needs to stay in D0, so check skip_bus_pm to avoid
893 * putting it into a low-power state in that case.
894 */
895 if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev))
896 pci_prepare_to_sleep(pci_dev);
897 }
898
899 pci_dbg(pci_dev, "PCI PM: Suspend power state: %s\n",
900 pci_power_name(pci_dev->current_state));
901
902 if (pci_dev->current_state == PCI_D0) {
903 pci_dev->skip_bus_pm = true;
904 /*
905 * Per PCI PM r1.2, table 6-1, a bridge must be in D0 if any
906 * downstream device is in D0, so avoid changing the power state
907 * of the parent bridge by setting the skip_bus_pm flag for it.
908 */
909 if (pci_dev->bus->self)
910 pci_dev->bus->self->skip_bus_pm = true;
911 }
912
913 if (pci_dev->skip_bus_pm && pm_suspend_no_platform()) {
914 pci_dbg(pci_dev, "PCI PM: Skipped\n");
915 goto Fixup;
916 }
917
918 pci_pm_set_unknown_state(pci_dev);
919
920 /*
921 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
922 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
923 * hasn't been quiesced and tries to turn it off. If the controller
924 * is already in D3, this can hang or cause memory corruption.
925 *
926 * Since the value of the COMMAND register doesn't matter once the
927 * device has been suspended, we can safely set it to 0 here.
928 */
929 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
930 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
931
932Fixup:
933 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
934
935 /*
936 * If the target system sleep state is suspend-to-idle, it is sufficient
937 * to check whether or not the device's wakeup settings are good for
938 * runtime PM. Otherwise, the pm_resume_via_firmware() check will cause
939 * pci_pm_complete() to take care of fixing up the device's state
940 * anyway, if need be.
941 */
942 if (device_can_wakeup(dev) && !device_may_wakeup(dev))
943 dev->power.may_skip_resume = false;
944
945 return 0;
946}
947
948static int pci_pm_resume_noirq(struct device *dev)
949{
950 struct pci_dev *pci_dev = to_pci_dev(dev);
951 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
952 pci_power_t prev_state = pci_dev->current_state;
953 bool skip_bus_pm = pci_dev->skip_bus_pm;
954
955 if (dev_pm_skip_resume(dev))
956 return 0;
957
958 /*
959 * In the suspend-to-idle case, devices left in D0 during suspend will
960 * stay in D0, so it is not necessary to restore or update their
961 * configuration here and attempting to put them into D0 again is
962 * pointless, so avoid doing that.
963 */
964 if (!(skip_bus_pm && pm_suspend_no_platform()))
965 pci_pm_default_resume_early(pci_dev);
966
967 pci_fixup_device(pci_fixup_resume_early, pci_dev);
968 pcie_pme_root_status_cleanup(pci_dev);
969
970 if (!skip_bus_pm && prev_state == PCI_D3cold)
971 pci_pm_bridge_power_up_actions(pci_dev);
972
973 if (pci_has_legacy_pm_support(pci_dev))
974 return 0;
975
976 if (pm && pm->resume_noirq)
977 return pm->resume_noirq(dev);
978
979 return 0;
980}
981
982static int pci_pm_resume_early(struct device *dev)
983{
984 if (dev_pm_skip_resume(dev))
985 return 0;
986
987 return pm_generic_resume_early(dev);
988}
989
990static int pci_pm_resume(struct device *dev)
991{
992 struct pci_dev *pci_dev = to_pci_dev(dev);
993 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
994
995 /*
996 * This is necessary for the suspend error path in which resume is
997 * called without restoring the standard config registers of the device.
998 */
999 if (pci_dev->state_saved)
1000 pci_restore_standard_config(pci_dev);
1001
1002 pci_resume_ptm(pci_dev);
1003
1004 if (pci_has_legacy_pm_support(pci_dev))
1005 return pci_legacy_resume(dev);
1006
1007 pci_pm_default_resume(pci_dev);
1008
1009 if (pm) {
1010 if (pm->resume)
1011 return pm->resume(dev);
1012 } else {
1013 pci_pm_reenable_device(pci_dev);
1014 }
1015
1016 return 0;
1017}
1018
1019#else /* !CONFIG_SUSPEND */
1020
1021#define pci_pm_suspend NULL
1022#define pci_pm_suspend_late NULL
1023#define pci_pm_suspend_noirq NULL
1024#define pci_pm_resume NULL
1025#define pci_pm_resume_early NULL
1026#define pci_pm_resume_noirq NULL
1027
1028#endif /* !CONFIG_SUSPEND */
1029
1030#ifdef CONFIG_HIBERNATE_CALLBACKS
1031
1032static int pci_pm_freeze(struct device *dev)
1033{
1034 struct pci_dev *pci_dev = to_pci_dev(dev);
1035 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1036
1037 if (pci_has_legacy_pm_support(pci_dev))
1038 return pci_legacy_suspend(dev, PMSG_FREEZE);
1039
1040 if (!pm) {
1041 pci_pm_default_suspend(pci_dev);
1042 return 0;
1043 }
1044
1045 /*
1046 * Resume all runtime-suspended devices before creating a snapshot
1047 * image of system memory, because the restore kernel generally cannot
1048 * be expected to always handle them consistently and they need to be
1049 * put into the runtime-active metastate during system resume anyway,
1050 * so it is better to ensure that the state saved in the image will be
1051 * always consistent with that.
1052 */
1053 pm_runtime_resume(dev);
1054 pci_dev->state_saved = false;
1055
1056 if (pm->freeze) {
1057 int error;
1058
1059 error = pm->freeze(dev);
1060 suspend_report_result(dev, pm->freeze, error);
1061 if (error)
1062 return error;
1063 }
1064
1065 return 0;
1066}
1067
1068static int pci_pm_freeze_noirq(struct device *dev)
1069{
1070 struct pci_dev *pci_dev = to_pci_dev(dev);
1071 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1072
1073 if (pci_has_legacy_pm_support(pci_dev))
1074 return pci_legacy_suspend_late(dev);
1075
1076 if (pm && pm->freeze_noirq) {
1077 int error;
1078
1079 error = pm->freeze_noirq(dev);
1080 suspend_report_result(dev, pm->freeze_noirq, error);
1081 if (error)
1082 return error;
1083 }
1084
1085 if (!pci_dev->state_saved)
1086 pci_save_state(pci_dev);
1087
1088 pci_pm_set_unknown_state(pci_dev);
1089
1090 return 0;
1091}
1092
1093static int pci_pm_thaw_noirq(struct device *dev)
1094{
1095 struct pci_dev *pci_dev = to_pci_dev(dev);
1096 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1097
1098 /*
1099 * The pm->thaw_noirq() callback assumes the device has been
1100 * returned to D0 and its config state has been restored.
1101 *
1102 * In addition, pci_restore_state() restores MSI-X state in MMIO
1103 * space, which requires the device to be in D0, so return it to D0
1104 * in case the driver's "freeze" callbacks put it into a low-power
1105 * state.
1106 */
1107 pci_pm_power_up_and_verify_state(pci_dev);
1108 pci_restore_state(pci_dev);
1109
1110 if (pci_has_legacy_pm_support(pci_dev))
1111 return 0;
1112
1113 if (pm && pm->thaw_noirq)
1114 return pm->thaw_noirq(dev);
1115
1116 return 0;
1117}
1118
1119static int pci_pm_thaw(struct device *dev)
1120{
1121 struct pci_dev *pci_dev = to_pci_dev(dev);
1122 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1123 int error = 0;
1124
1125 if (pci_has_legacy_pm_support(pci_dev))
1126 return pci_legacy_resume(dev);
1127
1128 if (pm) {
1129 if (pm->thaw)
1130 error = pm->thaw(dev);
1131 } else {
1132 pci_pm_reenable_device(pci_dev);
1133 }
1134
1135 pci_dev->state_saved = false;
1136
1137 return error;
1138}
1139
1140static int pci_pm_poweroff(struct device *dev)
1141{
1142 struct pci_dev *pci_dev = to_pci_dev(dev);
1143 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1144
1145 if (pci_has_legacy_pm_support(pci_dev))
1146 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1147
1148 if (!pm) {
1149 pci_pm_default_suspend(pci_dev);
1150 return 0;
1151 }
1152
1153 /* The reason to do that is the same as in pci_pm_suspend(). */
1154 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1155 pci_dev_need_resume(pci_dev)) {
1156 pm_runtime_resume(dev);
1157 pci_dev->state_saved = false;
1158 } else {
1159 pci_dev_adjust_pme(pci_dev);
1160 }
1161
1162 if (pm->poweroff) {
1163 int error;
1164
1165 error = pm->poweroff(dev);
1166 suspend_report_result(dev, pm->poweroff, error);
1167 if (error)
1168 return error;
1169 }
1170
1171 return 0;
1172}
1173
1174static int pci_pm_poweroff_late(struct device *dev)
1175{
1176 if (dev_pm_skip_suspend(dev))
1177 return 0;
1178
1179 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
1180
1181 return pm_generic_poweroff_late(dev);
1182}
1183
1184static int pci_pm_poweroff_noirq(struct device *dev)
1185{
1186 struct pci_dev *pci_dev = to_pci_dev(dev);
1187 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1188
1189 if (dev_pm_skip_suspend(dev))
1190 return 0;
1191
1192 if (pci_has_legacy_pm_support(pci_dev))
1193 return pci_legacy_suspend_late(dev);
1194
1195 if (!pm) {
1196 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1197 return 0;
1198 }
1199
1200 if (pm->poweroff_noirq) {
1201 int error;
1202
1203 error = pm->poweroff_noirq(dev);
1204 suspend_report_result(dev, pm->poweroff_noirq, error);
1205 if (error)
1206 return error;
1207 }
1208
1209 if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1210 pci_prepare_to_sleep(pci_dev);
1211
1212 /*
1213 * The reason for doing this here is the same as for the analogous code
1214 * in pci_pm_suspend_noirq().
1215 */
1216 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1217 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1218
1219 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1220
1221 return 0;
1222}
1223
1224static int pci_pm_restore_noirq(struct device *dev)
1225{
1226 struct pci_dev *pci_dev = to_pci_dev(dev);
1227 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1228
1229 pci_pm_default_resume_early(pci_dev);
1230 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1231
1232 if (pci_has_legacy_pm_support(pci_dev))
1233 return 0;
1234
1235 if (pm && pm->restore_noirq)
1236 return pm->restore_noirq(dev);
1237
1238 return 0;
1239}
1240
1241static int pci_pm_restore(struct device *dev)
1242{
1243 struct pci_dev *pci_dev = to_pci_dev(dev);
1244 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1245
1246 /*
1247 * This is necessary for the hibernation error path in which restore is
1248 * called without restoring the standard config registers of the device.
1249 */
1250 if (pci_dev->state_saved)
1251 pci_restore_standard_config(pci_dev);
1252
1253 if (pci_has_legacy_pm_support(pci_dev))
1254 return pci_legacy_resume(dev);
1255
1256 pci_pm_default_resume(pci_dev);
1257
1258 if (pm) {
1259 if (pm->restore)
1260 return pm->restore(dev);
1261 } else {
1262 pci_pm_reenable_device(pci_dev);
1263 }
1264
1265 return 0;
1266}
1267
1268#else /* !CONFIG_HIBERNATE_CALLBACKS */
1269
1270#define pci_pm_freeze NULL
1271#define pci_pm_freeze_noirq NULL
1272#define pci_pm_thaw NULL
1273#define pci_pm_thaw_noirq NULL
1274#define pci_pm_poweroff NULL
1275#define pci_pm_poweroff_late NULL
1276#define pci_pm_poweroff_noirq NULL
1277#define pci_pm_restore NULL
1278#define pci_pm_restore_noirq NULL
1279
1280#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1281
1282#ifdef CONFIG_PM
1283
1284static int pci_pm_runtime_suspend(struct device *dev)
1285{
1286 struct pci_dev *pci_dev = to_pci_dev(dev);
1287 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1288 pci_power_t prev = pci_dev->current_state;
1289 int error;
1290
1291 pci_suspend_ptm(pci_dev);
1292
1293 /*
1294 * If pci_dev->driver is not set (unbound), we leave the device in D0,
1295 * but it may go to D3cold when the bridge above it runtime suspends.
1296 * Save its config space in case that happens.
1297 */
1298 if (!pci_dev->driver) {
1299 pci_save_state(pci_dev);
1300 return 0;
1301 }
1302
1303 pci_dev->state_saved = false;
1304 if (pm && pm->runtime_suspend) {
1305 error = pm->runtime_suspend(dev);
1306 /*
1307 * -EBUSY and -EAGAIN is used to request the runtime PM core
1308 * to schedule a new suspend, so log the event only with debug
1309 * log level.
1310 */
1311 if (error == -EBUSY || error == -EAGAIN) {
1312 pci_dbg(pci_dev, "can't suspend now (%ps returned %d)\n",
1313 pm->runtime_suspend, error);
1314 return error;
1315 } else if (error) {
1316 pci_err(pci_dev, "can't suspend (%ps returned %d)\n",
1317 pm->runtime_suspend, error);
1318 return error;
1319 }
1320 }
1321
1322 pci_fixup_device(pci_fixup_suspend, pci_dev);
1323
1324 if (pm && pm->runtime_suspend
1325 && !pci_dev->state_saved && pci_dev->current_state != PCI_D0
1326 && pci_dev->current_state != PCI_UNKNOWN) {
1327 pci_WARN_ONCE(pci_dev, pci_dev->current_state != prev,
1328 "PCI PM: State of device not saved by %pS\n",
1329 pm->runtime_suspend);
1330 return 0;
1331 }
1332
1333 if (!pci_dev->state_saved) {
1334 pci_save_state(pci_dev);
1335 pci_finish_runtime_suspend(pci_dev);
1336 }
1337
1338 return 0;
1339}
1340
1341static int pci_pm_runtime_resume(struct device *dev)
1342{
1343 struct pci_dev *pci_dev = to_pci_dev(dev);
1344 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1345 pci_power_t prev_state = pci_dev->current_state;
1346 int error = 0;
1347
1348 /*
1349 * Restoring config space is necessary even if the device is not bound
1350 * to a driver because although we left it in D0, it may have gone to
1351 * D3cold when the bridge above it runtime suspended.
1352 */
1353 pci_pm_default_resume_early(pci_dev);
1354 pci_resume_ptm(pci_dev);
1355
1356 if (!pci_dev->driver)
1357 return 0;
1358
1359 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1360 pci_pm_default_resume(pci_dev);
1361
1362 if (prev_state == PCI_D3cold)
1363 pci_pm_bridge_power_up_actions(pci_dev);
1364
1365 if (pm && pm->runtime_resume)
1366 error = pm->runtime_resume(dev);
1367
1368 return error;
1369}
1370
1371static int pci_pm_runtime_idle(struct device *dev)
1372{
1373 struct pci_dev *pci_dev = to_pci_dev(dev);
1374 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1375
1376 /*
1377 * If pci_dev->driver is not set (unbound), the device should
1378 * always remain in D0 regardless of the runtime PM status
1379 */
1380 if (!pci_dev->driver)
1381 return 0;
1382
1383 if (pm && pm->runtime_idle)
1384 return pm->runtime_idle(dev);
1385
1386 return 0;
1387}
1388
1389static const struct dev_pm_ops pci_dev_pm_ops = {
1390 .prepare = pci_pm_prepare,
1391 .complete = pci_pm_complete,
1392 .suspend = pci_pm_suspend,
1393 .suspend_late = pci_pm_suspend_late,
1394 .resume = pci_pm_resume,
1395 .resume_early = pci_pm_resume_early,
1396 .freeze = pci_pm_freeze,
1397 .thaw = pci_pm_thaw,
1398 .poweroff = pci_pm_poweroff,
1399 .poweroff_late = pci_pm_poweroff_late,
1400 .restore = pci_pm_restore,
1401 .suspend_noirq = pci_pm_suspend_noirq,
1402 .resume_noirq = pci_pm_resume_noirq,
1403 .freeze_noirq = pci_pm_freeze_noirq,
1404 .thaw_noirq = pci_pm_thaw_noirq,
1405 .poweroff_noirq = pci_pm_poweroff_noirq,
1406 .restore_noirq = pci_pm_restore_noirq,
1407 .runtime_suspend = pci_pm_runtime_suspend,
1408 .runtime_resume = pci_pm_runtime_resume,
1409 .runtime_idle = pci_pm_runtime_idle,
1410};
1411
1412#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1413
1414#else /* !CONFIG_PM */
1415
1416#define pci_pm_runtime_suspend NULL
1417#define pci_pm_runtime_resume NULL
1418#define pci_pm_runtime_idle NULL
1419
1420#define PCI_PM_OPS_PTR NULL
1421
1422#endif /* !CONFIG_PM */
1423
1424/**
1425 * __pci_register_driver - register a new pci driver
1426 * @drv: the driver structure to register
1427 * @owner: owner module of drv
1428 * @mod_name: module name string
1429 *
1430 * Adds the driver structure to the list of registered drivers.
1431 * Returns a negative value on error, otherwise 0.
1432 * If no error occurred, the driver remains registered even if
1433 * no device was claimed during registration.
1434 */
1435int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1436 const char *mod_name)
1437{
1438 /* initialize common driver fields */
1439 drv->driver.name = drv->name;
1440 drv->driver.bus = &pci_bus_type;
1441 drv->driver.owner = owner;
1442 drv->driver.mod_name = mod_name;
1443 drv->driver.groups = drv->groups;
1444 drv->driver.dev_groups = drv->dev_groups;
1445
1446 spin_lock_init(&drv->dynids.lock);
1447 INIT_LIST_HEAD(&drv->dynids.list);
1448
1449 /* register with core */
1450 return driver_register(&drv->driver);
1451}
1452EXPORT_SYMBOL(__pci_register_driver);
1453
1454/**
1455 * pci_unregister_driver - unregister a pci driver
1456 * @drv: the driver structure to unregister
1457 *
1458 * Deletes the driver structure from the list of registered PCI drivers,
1459 * gives it a chance to clean up by calling its remove() function for
1460 * each device it was responsible for, and marks those devices as
1461 * driverless.
1462 */
1463
1464void pci_unregister_driver(struct pci_driver *drv)
1465{
1466 driver_unregister(&drv->driver);
1467 pci_free_dynids(drv);
1468}
1469EXPORT_SYMBOL(pci_unregister_driver);
1470
1471static struct pci_driver pci_compat_driver = {
1472 .name = "compat"
1473};
1474
1475/**
1476 * pci_dev_driver - get the pci_driver of a device
1477 * @dev: the device to query
1478 *
1479 * Returns the appropriate pci_driver structure or %NULL if there is no
1480 * registered driver for the device.
1481 */
1482struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1483{
1484 int i;
1485
1486 if (dev->driver)
1487 return dev->driver;
1488
1489 for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1490 if (dev->resource[i].flags & IORESOURCE_BUSY)
1491 return &pci_compat_driver;
1492
1493 return NULL;
1494}
1495EXPORT_SYMBOL(pci_dev_driver);
1496
1497/**
1498 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1499 * @dev: the PCI device structure to match against
1500 * @drv: the device driver to search for matching PCI device id structures
1501 *
1502 * Used by a driver to check whether a PCI device present in the
1503 * system is in its list of supported devices. Returns the matching
1504 * pci_device_id structure or %NULL if there is no match.
1505 */
1506static int pci_bus_match(struct device *dev, const struct device_driver *drv)
1507{
1508 struct pci_dev *pci_dev = to_pci_dev(dev);
1509 struct pci_driver *pci_drv;
1510 const struct pci_device_id *found_id;
1511
1512 if (!pci_dev->match_driver)
1513 return 0;
1514
1515 pci_drv = (struct pci_driver *)to_pci_driver(drv);
1516 found_id = pci_match_device(pci_drv, pci_dev);
1517 if (found_id)
1518 return 1;
1519
1520 return 0;
1521}
1522
1523/**
1524 * pci_dev_get - increments the reference count of the pci device structure
1525 * @dev: the device being referenced
1526 *
1527 * Each live reference to a device should be refcounted.
1528 *
1529 * Drivers for PCI devices should normally record such references in
1530 * their probe() methods, when they bind to a device, and release
1531 * them by calling pci_dev_put(), in their disconnect() methods.
1532 *
1533 * A pointer to the device with the incremented reference counter is returned.
1534 */
1535struct pci_dev *pci_dev_get(struct pci_dev *dev)
1536{
1537 if (dev)
1538 get_device(&dev->dev);
1539 return dev;
1540}
1541EXPORT_SYMBOL(pci_dev_get);
1542
1543/**
1544 * pci_dev_put - release a use of the pci device structure
1545 * @dev: device that's been disconnected
1546 *
1547 * Must be called when a user of a device is finished with it. When the last
1548 * user of the device calls this function, the memory of the device is freed.
1549 */
1550void pci_dev_put(struct pci_dev *dev)
1551{
1552 if (dev)
1553 put_device(&dev->dev);
1554}
1555EXPORT_SYMBOL(pci_dev_put);
1556
1557static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env)
1558{
1559 const struct pci_dev *pdev;
1560
1561 if (!dev)
1562 return -ENODEV;
1563
1564 pdev = to_pci_dev(dev);
1565
1566 if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1567 return -ENOMEM;
1568
1569 if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1570 return -ENOMEM;
1571
1572 if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1573 pdev->subsystem_device))
1574 return -ENOMEM;
1575
1576 if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1577 return -ENOMEM;
1578
1579 if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1580 pdev->vendor, pdev->device,
1581 pdev->subsystem_vendor, pdev->subsystem_device,
1582 (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1583 (u8)(pdev->class)))
1584 return -ENOMEM;
1585
1586 return 0;
1587}
1588
1589#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH)
1590/**
1591 * pci_uevent_ers - emit a uevent during recovery path of PCI device
1592 * @pdev: PCI device undergoing error recovery
1593 * @err_type: type of error event
1594 */
1595void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1596{
1597 int idx = 0;
1598 char *envp[3];
1599
1600 switch (err_type) {
1601 case PCI_ERS_RESULT_NONE:
1602 case PCI_ERS_RESULT_CAN_RECOVER:
1603 envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1604 envp[idx++] = "DEVICE_ONLINE=0";
1605 break;
1606 case PCI_ERS_RESULT_RECOVERED:
1607 envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1608 envp[idx++] = "DEVICE_ONLINE=1";
1609 break;
1610 case PCI_ERS_RESULT_DISCONNECT:
1611 envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1612 envp[idx++] = "DEVICE_ONLINE=0";
1613 break;
1614 default:
1615 break;
1616 }
1617
1618 if (idx > 0) {
1619 envp[idx++] = NULL;
1620 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1621 }
1622}
1623#endif
1624
1625static int pci_bus_num_vf(struct device *dev)
1626{
1627 return pci_num_vf(to_pci_dev(dev));
1628}
1629
1630/**
1631 * pci_dma_configure - Setup DMA configuration
1632 * @dev: ptr to dev structure
1633 *
1634 * Function to update PCI devices's DMA configuration using the same
1635 * info from the OF node or ACPI node of host bridge's parent (if any).
1636 */
1637static int pci_dma_configure(struct device *dev)
1638{
1639 struct pci_driver *driver = to_pci_driver(dev->driver);
1640 struct device *bridge;
1641 int ret = 0;
1642
1643 bridge = pci_get_host_bridge_device(to_pci_dev(dev));
1644
1645 if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
1646 bridge->parent->of_node) {
1647 ret = of_dma_configure(dev, bridge->parent->of_node, true);
1648 } else if (has_acpi_companion(bridge)) {
1649 struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
1650
1651 ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev));
1652 }
1653
1654 pci_put_host_bridge_device(bridge);
1655
1656 if (!ret && !driver->driver_managed_dma) {
1657 ret = iommu_device_use_default_domain(dev);
1658 if (ret)
1659 arch_teardown_dma_ops(dev);
1660 }
1661
1662 return ret;
1663}
1664
1665static void pci_dma_cleanup(struct device *dev)
1666{
1667 struct pci_driver *driver = to_pci_driver(dev->driver);
1668
1669 if (!driver->driver_managed_dma)
1670 iommu_device_unuse_default_domain(dev);
1671}
1672
1673const struct bus_type pci_bus_type = {
1674 .name = "pci",
1675 .match = pci_bus_match,
1676 .uevent = pci_uevent,
1677 .probe = pci_device_probe,
1678 .remove = pci_device_remove,
1679 .shutdown = pci_device_shutdown,
1680 .dev_groups = pci_dev_groups,
1681 .bus_groups = pci_bus_groups,
1682 .drv_groups = pci_drv_groups,
1683 .pm = PCI_PM_OPS_PTR,
1684 .num_vf = pci_bus_num_vf,
1685 .dma_configure = pci_dma_configure,
1686 .dma_cleanup = pci_dma_cleanup,
1687};
1688EXPORT_SYMBOL(pci_bus_type);
1689
1690#ifdef CONFIG_PCIEPORTBUS
1691static int pcie_port_bus_match(struct device *dev, const struct device_driver *drv)
1692{
1693 struct pcie_device *pciedev;
1694 const struct pcie_port_service_driver *driver;
1695
1696 if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1697 return 0;
1698
1699 pciedev = to_pcie_device(dev);
1700 driver = to_service_driver(drv);
1701
1702 if (driver->service != pciedev->service)
1703 return 0;
1704
1705 if (driver->port_type != PCIE_ANY_PORT &&
1706 driver->port_type != pci_pcie_type(pciedev->port))
1707 return 0;
1708
1709 return 1;
1710}
1711
1712const struct bus_type pcie_port_bus_type = {
1713 .name = "pci_express",
1714 .match = pcie_port_bus_match,
1715};
1716#endif
1717
1718static int __init pci_driver_init(void)
1719{
1720 int ret;
1721
1722 ret = bus_register(&pci_bus_type);
1723 if (ret)
1724 return ret;
1725
1726#ifdef CONFIG_PCIEPORTBUS
1727 ret = bus_register(&pcie_port_bus_type);
1728 if (ret)
1729 return ret;
1730#endif
1731 dma_debug_add_bus(&pci_bus_type);
1732 return 0;
1733}
1734postcore_initcall(pci_driver_init);