Loading...
1/*
2 * drivers/pci/pci-driver.c
3 *
4 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
5 * (C) Copyright 2007 Novell Inc.
6 *
7 * Released under the GPL v2 only.
8 *
9 */
10
11#include <linux/pci.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/device.h>
15#include <linux/mempolicy.h>
16#include <linux/string.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/cpu.h>
20#include <linux/pm_runtime.h>
21#include <linux/suspend.h>
22#include "pci.h"
23
24struct pci_dynid {
25 struct list_head node;
26 struct pci_device_id id;
27};
28
29/**
30 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
31 * @drv: target pci driver
32 * @vendor: PCI vendor ID
33 * @device: PCI device ID
34 * @subvendor: PCI subvendor ID
35 * @subdevice: PCI subdevice ID
36 * @class: PCI class
37 * @class_mask: PCI class mask
38 * @driver_data: private driver data
39 *
40 * Adds a new dynamic pci device ID to this driver and causes the
41 * driver to probe for all devices again. @drv must have been
42 * registered prior to calling this function.
43 *
44 * CONTEXT:
45 * Does GFP_KERNEL allocation.
46 *
47 * RETURNS:
48 * 0 on success, -errno on failure.
49 */
50int pci_add_dynid(struct pci_driver *drv,
51 unsigned int vendor, unsigned int device,
52 unsigned int subvendor, unsigned int subdevice,
53 unsigned int class, unsigned int class_mask,
54 unsigned long driver_data)
55{
56 struct pci_dynid *dynid;
57 int retval;
58
59 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
60 if (!dynid)
61 return -ENOMEM;
62
63 dynid->id.vendor = vendor;
64 dynid->id.device = device;
65 dynid->id.subvendor = subvendor;
66 dynid->id.subdevice = subdevice;
67 dynid->id.class = class;
68 dynid->id.class_mask = class_mask;
69 dynid->id.driver_data = driver_data;
70
71 spin_lock(&drv->dynids.lock);
72 list_add_tail(&dynid->node, &drv->dynids.list);
73 spin_unlock(&drv->dynids.lock);
74
75 retval = driver_attach(&drv->driver);
76
77 return retval;
78}
79
80static void pci_free_dynids(struct pci_driver *drv)
81{
82 struct pci_dynid *dynid, *n;
83
84 spin_lock(&drv->dynids.lock);
85 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
86 list_del(&dynid->node);
87 kfree(dynid);
88 }
89 spin_unlock(&drv->dynids.lock);
90}
91
92/*
93 * Dynamic device ID manipulation via sysfs is disabled for !CONFIG_HOTPLUG
94 */
95#ifdef CONFIG_HOTPLUG
96/**
97 * store_new_id - sysfs frontend to pci_add_dynid()
98 * @driver: target device driver
99 * @buf: buffer for scanning device ID data
100 * @count: input size
101 *
102 * Allow PCI IDs to be added to an existing driver via sysfs.
103 */
104static ssize_t
105store_new_id(struct device_driver *driver, const char *buf, size_t count)
106{
107 struct pci_driver *pdrv = to_pci_driver(driver);
108 const struct pci_device_id *ids = pdrv->id_table;
109 __u32 vendor, device, subvendor=PCI_ANY_ID,
110 subdevice=PCI_ANY_ID, class=0, class_mask=0;
111 unsigned long driver_data=0;
112 int fields=0;
113 int retval;
114
115 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
116 &vendor, &device, &subvendor, &subdevice,
117 &class, &class_mask, &driver_data);
118 if (fields < 2)
119 return -EINVAL;
120
121 /* Only accept driver_data values that match an existing id_table
122 entry */
123 if (ids) {
124 retval = -EINVAL;
125 while (ids->vendor || ids->subvendor || ids->class_mask) {
126 if (driver_data == ids->driver_data) {
127 retval = 0;
128 break;
129 }
130 ids++;
131 }
132 if (retval) /* No match */
133 return retval;
134 }
135
136 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
137 class, class_mask, driver_data);
138 if (retval)
139 return retval;
140 return count;
141}
142static DRIVER_ATTR(new_id, S_IWUSR, NULL, store_new_id);
143
144/**
145 * store_remove_id - remove a PCI device ID from this driver
146 * @driver: target device driver
147 * @buf: buffer for scanning device ID data
148 * @count: input size
149 *
150 * Removes a dynamic pci device ID to this driver.
151 */
152static ssize_t
153store_remove_id(struct device_driver *driver, const char *buf, size_t count)
154{
155 struct pci_dynid *dynid, *n;
156 struct pci_driver *pdrv = to_pci_driver(driver);
157 __u32 vendor, device, subvendor = PCI_ANY_ID,
158 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
159 int fields = 0;
160 int retval = -ENODEV;
161
162 fields = sscanf(buf, "%x %x %x %x %x %x",
163 &vendor, &device, &subvendor, &subdevice,
164 &class, &class_mask);
165 if (fields < 2)
166 return -EINVAL;
167
168 spin_lock(&pdrv->dynids.lock);
169 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
170 struct pci_device_id *id = &dynid->id;
171 if ((id->vendor == vendor) &&
172 (id->device == device) &&
173 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
174 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
175 !((id->class ^ class) & class_mask)) {
176 list_del(&dynid->node);
177 kfree(dynid);
178 retval = 0;
179 break;
180 }
181 }
182 spin_unlock(&pdrv->dynids.lock);
183
184 if (retval)
185 return retval;
186 return count;
187}
188static DRIVER_ATTR(remove_id, S_IWUSR, NULL, store_remove_id);
189
190static int
191pci_create_newid_files(struct pci_driver *drv)
192{
193 int error = 0;
194
195 if (drv->probe != NULL) {
196 error = driver_create_file(&drv->driver, &driver_attr_new_id);
197 if (error == 0) {
198 error = driver_create_file(&drv->driver,
199 &driver_attr_remove_id);
200 if (error)
201 driver_remove_file(&drv->driver,
202 &driver_attr_new_id);
203 }
204 }
205 return error;
206}
207
208static void pci_remove_newid_files(struct pci_driver *drv)
209{
210 driver_remove_file(&drv->driver, &driver_attr_remove_id);
211 driver_remove_file(&drv->driver, &driver_attr_new_id);
212}
213#else /* !CONFIG_HOTPLUG */
214static inline int pci_create_newid_files(struct pci_driver *drv)
215{
216 return 0;
217}
218static inline void pci_remove_newid_files(struct pci_driver *drv) {}
219#endif
220
221/**
222 * pci_match_id - See if a pci device matches a given pci_id table
223 * @ids: array of PCI device id structures to search in
224 * @dev: the PCI device structure to match against.
225 *
226 * Used by a driver to check whether a PCI device present in the
227 * system is in its list of supported devices. Returns the matching
228 * pci_device_id structure or %NULL if there is no match.
229 *
230 * Deprecated, don't use this as it will not catch any dynamic ids
231 * that a driver might want to check for.
232 */
233const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
234 struct pci_dev *dev)
235{
236 if (ids) {
237 while (ids->vendor || ids->subvendor || ids->class_mask) {
238 if (pci_match_one_device(ids, dev))
239 return ids;
240 ids++;
241 }
242 }
243 return NULL;
244}
245
246/**
247 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
248 * @drv: the PCI driver to match against
249 * @dev: the PCI device structure to match against
250 *
251 * Used by a driver to check whether a PCI device present in the
252 * system is in its list of supported devices. Returns the matching
253 * pci_device_id structure or %NULL if there is no match.
254 */
255static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
256 struct pci_dev *dev)
257{
258 struct pci_dynid *dynid;
259
260 /* Look at the dynamic ids first, before the static ones */
261 spin_lock(&drv->dynids.lock);
262 list_for_each_entry(dynid, &drv->dynids.list, node) {
263 if (pci_match_one_device(&dynid->id, dev)) {
264 spin_unlock(&drv->dynids.lock);
265 return &dynid->id;
266 }
267 }
268 spin_unlock(&drv->dynids.lock);
269
270 return pci_match_id(drv->id_table, dev);
271}
272
273struct drv_dev_and_id {
274 struct pci_driver *drv;
275 struct pci_dev *dev;
276 const struct pci_device_id *id;
277};
278
279static long local_pci_probe(void *_ddi)
280{
281 struct drv_dev_and_id *ddi = _ddi;
282 struct device *dev = &ddi->dev->dev;
283 int rc;
284
285 /* Unbound PCI devices are always set to disabled and suspended.
286 * During probe, the device is set to enabled and active and the
287 * usage count is incremented. If the driver supports runtime PM,
288 * it should call pm_runtime_put_noidle() in its probe routine and
289 * pm_runtime_get_noresume() in its remove routine.
290 */
291 pm_runtime_get_noresume(dev);
292 pm_runtime_set_active(dev);
293 pm_runtime_enable(dev);
294
295 rc = ddi->drv->probe(ddi->dev, ddi->id);
296 if (rc) {
297 pm_runtime_disable(dev);
298 pm_runtime_set_suspended(dev);
299 pm_runtime_put_noidle(dev);
300 }
301 return rc;
302}
303
304static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
305 const struct pci_device_id *id)
306{
307 int error, node;
308 struct drv_dev_and_id ddi = { drv, dev, id };
309
310 /* Execute driver initialization on node where the device's
311 bus is attached to. This way the driver likely allocates
312 its local memory on the right node without any need to
313 change it. */
314 node = dev_to_node(&dev->dev);
315 if (node >= 0) {
316 int cpu;
317
318 get_online_cpus();
319 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
320 if (cpu < nr_cpu_ids)
321 error = work_on_cpu(cpu, local_pci_probe, &ddi);
322 else
323 error = local_pci_probe(&ddi);
324 put_online_cpus();
325 } else
326 error = local_pci_probe(&ddi);
327 return error;
328}
329
330/**
331 * __pci_device_probe - check if a driver wants to claim a specific PCI device
332 * @drv: driver to call to check if it wants the PCI device
333 * @pci_dev: PCI device being probed
334 *
335 * returns 0 on success, else error.
336 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
337 */
338static int
339__pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
340{
341 const struct pci_device_id *id;
342 int error = 0;
343
344 if (!pci_dev->driver && drv->probe) {
345 error = -ENODEV;
346
347 id = pci_match_device(drv, pci_dev);
348 if (id)
349 error = pci_call_probe(drv, pci_dev, id);
350 if (error >= 0) {
351 pci_dev->driver = drv;
352 error = 0;
353 }
354 }
355 return error;
356}
357
358static int pci_device_probe(struct device * dev)
359{
360 int error = 0;
361 struct pci_driver *drv;
362 struct pci_dev *pci_dev;
363
364 drv = to_pci_driver(dev->driver);
365 pci_dev = to_pci_dev(dev);
366 pci_dev_get(pci_dev);
367 error = __pci_device_probe(drv, pci_dev);
368 if (error)
369 pci_dev_put(pci_dev);
370
371 return error;
372}
373
374static int pci_device_remove(struct device * dev)
375{
376 struct pci_dev * pci_dev = to_pci_dev(dev);
377 struct pci_driver * drv = pci_dev->driver;
378
379 if (drv) {
380 if (drv->remove) {
381 pm_runtime_get_sync(dev);
382 drv->remove(pci_dev);
383 pm_runtime_put_noidle(dev);
384 }
385 pci_dev->driver = NULL;
386 }
387
388 /* Undo the runtime PM settings in local_pci_probe() */
389 pm_runtime_disable(dev);
390 pm_runtime_set_suspended(dev);
391 pm_runtime_put_noidle(dev);
392
393 /*
394 * If the device is still on, set the power state as "unknown",
395 * since it might change by the next time we load the driver.
396 */
397 if (pci_dev->current_state == PCI_D0)
398 pci_dev->current_state = PCI_UNKNOWN;
399
400 /*
401 * We would love to complain here if pci_dev->is_enabled is set, that
402 * the driver should have called pci_disable_device(), but the
403 * unfortunate fact is there are too many odd BIOS and bridge setups
404 * that don't like drivers doing that all of the time.
405 * Oh well, we can dream of sane hardware when we sleep, no matter how
406 * horrible the crap we have to deal with is when we are awake...
407 */
408
409 pci_dev_put(pci_dev);
410 return 0;
411}
412
413static void pci_device_shutdown(struct device *dev)
414{
415 struct pci_dev *pci_dev = to_pci_dev(dev);
416 struct pci_driver *drv = pci_dev->driver;
417
418 if (drv && drv->shutdown)
419 drv->shutdown(pci_dev);
420 pci_msi_shutdown(pci_dev);
421 pci_msix_shutdown(pci_dev);
422
423 /*
424 * Turn off Bus Master bit on the device to tell it to not
425 * continue to do DMA
426 */
427 pci_disable_device(pci_dev);
428
429 /*
430 * Devices may be enabled to wake up by runtime PM, but they need not
431 * be supposed to wake up the system from its "power off" state (e.g.
432 * ACPI S5). Therefore disable wakeup for all devices that aren't
433 * supposed to wake up the system at this point. The state argument
434 * will be ignored by pci_enable_wake().
435 */
436 if (!device_may_wakeup(dev))
437 pci_enable_wake(pci_dev, PCI_UNKNOWN, false);
438}
439
440#ifdef CONFIG_PM
441
442/* Auxiliary functions used for system resume and run-time resume. */
443
444/**
445 * pci_restore_standard_config - restore standard config registers of PCI device
446 * @pci_dev: PCI device to handle
447 */
448static int pci_restore_standard_config(struct pci_dev *pci_dev)
449{
450 pci_update_current_state(pci_dev, PCI_UNKNOWN);
451
452 if (pci_dev->current_state != PCI_D0) {
453 int error = pci_set_power_state(pci_dev, PCI_D0);
454 if (error)
455 return error;
456 }
457
458 pci_restore_state(pci_dev);
459 return 0;
460}
461
462static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
463{
464 pci_restore_standard_config(pci_dev);
465 pci_fixup_device(pci_fixup_resume_early, pci_dev);
466}
467
468#endif
469
470#ifdef CONFIG_PM_SLEEP
471
472/*
473 * Default "suspend" method for devices that have no driver provided suspend,
474 * or not even a driver at all (second part).
475 */
476static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
477{
478 /*
479 * mark its power state as "unknown", since we don't know if
480 * e.g. the BIOS will change its device state when we suspend.
481 */
482 if (pci_dev->current_state == PCI_D0)
483 pci_dev->current_state = PCI_UNKNOWN;
484}
485
486/*
487 * Default "resume" method for devices that have no driver provided resume,
488 * or not even a driver at all (second part).
489 */
490static int pci_pm_reenable_device(struct pci_dev *pci_dev)
491{
492 int retval;
493
494 /* if the device was enabled before suspend, reenable */
495 retval = pci_reenable_device(pci_dev);
496 /*
497 * if the device was busmaster before the suspend, make it busmaster
498 * again
499 */
500 if (pci_dev->is_busmaster)
501 pci_set_master(pci_dev);
502
503 return retval;
504}
505
506static int pci_legacy_suspend(struct device *dev, pm_message_t state)
507{
508 struct pci_dev * pci_dev = to_pci_dev(dev);
509 struct pci_driver * drv = pci_dev->driver;
510
511 if (drv && drv->suspend) {
512 pci_power_t prev = pci_dev->current_state;
513 int error;
514
515 error = drv->suspend(pci_dev, state);
516 suspend_report_result(drv->suspend, error);
517 if (error)
518 return error;
519
520 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
521 && pci_dev->current_state != PCI_UNKNOWN) {
522 WARN_ONCE(pci_dev->current_state != prev,
523 "PCI PM: Device state not saved by %pF\n",
524 drv->suspend);
525 }
526 }
527
528 pci_fixup_device(pci_fixup_suspend, pci_dev);
529
530 return 0;
531}
532
533static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
534{
535 struct pci_dev * pci_dev = to_pci_dev(dev);
536 struct pci_driver * drv = pci_dev->driver;
537
538 if (drv && drv->suspend_late) {
539 pci_power_t prev = pci_dev->current_state;
540 int error;
541
542 error = drv->suspend_late(pci_dev, state);
543 suspend_report_result(drv->suspend_late, error);
544 if (error)
545 return error;
546
547 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
548 && pci_dev->current_state != PCI_UNKNOWN) {
549 WARN_ONCE(pci_dev->current_state != prev,
550 "PCI PM: Device state not saved by %pF\n",
551 drv->suspend_late);
552 return 0;
553 }
554 }
555
556 if (!pci_dev->state_saved)
557 pci_save_state(pci_dev);
558
559 pci_pm_set_unknown_state(pci_dev);
560
561 return 0;
562}
563
564static int pci_legacy_resume_early(struct device *dev)
565{
566 struct pci_dev * pci_dev = to_pci_dev(dev);
567 struct pci_driver * drv = pci_dev->driver;
568
569 return drv && drv->resume_early ?
570 drv->resume_early(pci_dev) : 0;
571}
572
573static int pci_legacy_resume(struct device *dev)
574{
575 struct pci_dev * pci_dev = to_pci_dev(dev);
576 struct pci_driver * drv = pci_dev->driver;
577
578 pci_fixup_device(pci_fixup_resume, pci_dev);
579
580 return drv && drv->resume ?
581 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
582}
583
584/* Auxiliary functions used by the new power management framework */
585
586static void pci_pm_default_resume(struct pci_dev *pci_dev)
587{
588 pci_fixup_device(pci_fixup_resume, pci_dev);
589
590 if (!pci_is_bridge(pci_dev))
591 pci_enable_wake(pci_dev, PCI_D0, false);
592}
593
594static void pci_pm_default_suspend(struct pci_dev *pci_dev)
595{
596 /* Disable non-bridge devices without PM support */
597 if (!pci_is_bridge(pci_dev))
598 pci_disable_enabled_device(pci_dev);
599}
600
601static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
602{
603 struct pci_driver *drv = pci_dev->driver;
604 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
605 || drv->resume_early);
606
607 /*
608 * Legacy PM support is used by default, so warn if the new framework is
609 * supported as well. Drivers are supposed to support either the
610 * former, or the latter, but not both at the same time.
611 */
612 WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
613 drv->name, pci_dev->vendor, pci_dev->device);
614
615 return ret;
616}
617
618/* New power management framework */
619
620static int pci_pm_prepare(struct device *dev)
621{
622 struct device_driver *drv = dev->driver;
623 int error = 0;
624
625 /*
626 * If a PCI device configured to wake up the system from sleep states
627 * has been suspended at run time and there's a resume request pending
628 * for it, this is equivalent to the device signaling wakeup, so the
629 * system suspend operation should be aborted.
630 */
631 pm_runtime_get_noresume(dev);
632 if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
633 pm_wakeup_event(dev, 0);
634
635 if (pm_wakeup_pending()) {
636 pm_runtime_put_sync(dev);
637 return -EBUSY;
638 }
639
640 /*
641 * PCI devices suspended at run time need to be resumed at this
642 * point, because in general it is necessary to reconfigure them for
643 * system suspend. Namely, if the device is supposed to wake up the
644 * system from the sleep state, we may need to reconfigure it for this
645 * purpose. In turn, if the device is not supposed to wake up the
646 * system from the sleep state, we'll have to prevent it from signaling
647 * wake-up.
648 */
649 pm_runtime_resume(dev);
650
651 if (drv && drv->pm && drv->pm->prepare)
652 error = drv->pm->prepare(dev);
653
654 return error;
655}
656
657static void pci_pm_complete(struct device *dev)
658{
659 struct device_driver *drv = dev->driver;
660
661 if (drv && drv->pm && drv->pm->complete)
662 drv->pm->complete(dev);
663
664 pm_runtime_put_sync(dev);
665}
666
667#else /* !CONFIG_PM_SLEEP */
668
669#define pci_pm_prepare NULL
670#define pci_pm_complete NULL
671
672#endif /* !CONFIG_PM_SLEEP */
673
674#ifdef CONFIG_SUSPEND
675
676static int pci_pm_suspend(struct device *dev)
677{
678 struct pci_dev *pci_dev = to_pci_dev(dev);
679 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
680
681 if (pci_has_legacy_pm_support(pci_dev))
682 return pci_legacy_suspend(dev, PMSG_SUSPEND);
683
684 if (!pm) {
685 pci_pm_default_suspend(pci_dev);
686 goto Fixup;
687 }
688
689 if (pm->suspend) {
690 pci_power_t prev = pci_dev->current_state;
691 int error;
692
693 error = pm->suspend(dev);
694 suspend_report_result(pm->suspend, error);
695 if (error)
696 return error;
697
698 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
699 && pci_dev->current_state != PCI_UNKNOWN) {
700 WARN_ONCE(pci_dev->current_state != prev,
701 "PCI PM: State of device not saved by %pF\n",
702 pm->suspend);
703 }
704 }
705
706 Fixup:
707 pci_fixup_device(pci_fixup_suspend, pci_dev);
708
709 return 0;
710}
711
712static int pci_pm_suspend_noirq(struct device *dev)
713{
714 struct pci_dev *pci_dev = to_pci_dev(dev);
715 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
716
717 if (pci_has_legacy_pm_support(pci_dev))
718 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
719
720 if (!pm) {
721 pci_save_state(pci_dev);
722 return 0;
723 }
724
725 if (pm->suspend_noirq) {
726 pci_power_t prev = pci_dev->current_state;
727 int error;
728
729 error = pm->suspend_noirq(dev);
730 suspend_report_result(pm->suspend_noirq, error);
731 if (error)
732 return error;
733
734 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
735 && pci_dev->current_state != PCI_UNKNOWN) {
736 WARN_ONCE(pci_dev->current_state != prev,
737 "PCI PM: State of device not saved by %pF\n",
738 pm->suspend_noirq);
739 return 0;
740 }
741 }
742
743 if (!pci_dev->state_saved) {
744 pci_save_state(pci_dev);
745 if (!pci_is_bridge(pci_dev))
746 pci_prepare_to_sleep(pci_dev);
747 }
748
749 pci_pm_set_unknown_state(pci_dev);
750
751 /*
752 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
753 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
754 * hasn't been quiesced and tries to turn it off. If the controller
755 * is already in D3, this can hang or cause memory corruption.
756 *
757 * Since the value of the COMMAND register doesn't matter once the
758 * device has been suspended, we can safely set it to 0 here.
759 */
760 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
761 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
762
763 return 0;
764}
765
766static int pci_pm_resume_noirq(struct device *dev)
767{
768 struct pci_dev *pci_dev = to_pci_dev(dev);
769 struct device_driver *drv = dev->driver;
770 int error = 0;
771
772 pci_pm_default_resume_early(pci_dev);
773
774 if (pci_has_legacy_pm_support(pci_dev))
775 return pci_legacy_resume_early(dev);
776
777 if (drv && drv->pm && drv->pm->resume_noirq)
778 error = drv->pm->resume_noirq(dev);
779
780 return error;
781}
782
783static int pci_pm_resume(struct device *dev)
784{
785 struct pci_dev *pci_dev = to_pci_dev(dev);
786 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
787 int error = 0;
788
789 /*
790 * This is necessary for the suspend error path in which resume is
791 * called without restoring the standard config registers of the device.
792 */
793 if (pci_dev->state_saved)
794 pci_restore_standard_config(pci_dev);
795
796 if (pci_has_legacy_pm_support(pci_dev))
797 return pci_legacy_resume(dev);
798
799 pci_pm_default_resume(pci_dev);
800
801 if (pm) {
802 if (pm->resume)
803 error = pm->resume(dev);
804 } else {
805 pci_pm_reenable_device(pci_dev);
806 }
807
808 return error;
809}
810
811#else /* !CONFIG_SUSPEND */
812
813#define pci_pm_suspend NULL
814#define pci_pm_suspend_noirq NULL
815#define pci_pm_resume NULL
816#define pci_pm_resume_noirq NULL
817
818#endif /* !CONFIG_SUSPEND */
819
820#ifdef CONFIG_HIBERNATE_CALLBACKS
821
822static int pci_pm_freeze(struct device *dev)
823{
824 struct pci_dev *pci_dev = to_pci_dev(dev);
825 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
826
827 if (pci_has_legacy_pm_support(pci_dev))
828 return pci_legacy_suspend(dev, PMSG_FREEZE);
829
830 if (!pm) {
831 pci_pm_default_suspend(pci_dev);
832 return 0;
833 }
834
835 if (pm->freeze) {
836 int error;
837
838 error = pm->freeze(dev);
839 suspend_report_result(pm->freeze, error);
840 if (error)
841 return error;
842 }
843
844 return 0;
845}
846
847static int pci_pm_freeze_noirq(struct device *dev)
848{
849 struct pci_dev *pci_dev = to_pci_dev(dev);
850 struct device_driver *drv = dev->driver;
851
852 if (pci_has_legacy_pm_support(pci_dev))
853 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
854
855 if (drv && drv->pm && drv->pm->freeze_noirq) {
856 int error;
857
858 error = drv->pm->freeze_noirq(dev);
859 suspend_report_result(drv->pm->freeze_noirq, error);
860 if (error)
861 return error;
862 }
863
864 if (!pci_dev->state_saved)
865 pci_save_state(pci_dev);
866
867 pci_pm_set_unknown_state(pci_dev);
868
869 return 0;
870}
871
872static int pci_pm_thaw_noirq(struct device *dev)
873{
874 struct pci_dev *pci_dev = to_pci_dev(dev);
875 struct device_driver *drv = dev->driver;
876 int error = 0;
877
878 if (pci_has_legacy_pm_support(pci_dev))
879 return pci_legacy_resume_early(dev);
880
881 pci_update_current_state(pci_dev, PCI_D0);
882
883 if (drv && drv->pm && drv->pm->thaw_noirq)
884 error = drv->pm->thaw_noirq(dev);
885
886 return error;
887}
888
889static int pci_pm_thaw(struct device *dev)
890{
891 struct pci_dev *pci_dev = to_pci_dev(dev);
892 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
893 int error = 0;
894
895 if (pci_has_legacy_pm_support(pci_dev))
896 return pci_legacy_resume(dev);
897
898 if (pm) {
899 if (pm->thaw)
900 error = pm->thaw(dev);
901 } else {
902 pci_pm_reenable_device(pci_dev);
903 }
904
905 pci_dev->state_saved = false;
906
907 return error;
908}
909
910static int pci_pm_poweroff(struct device *dev)
911{
912 struct pci_dev *pci_dev = to_pci_dev(dev);
913 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
914
915 if (pci_has_legacy_pm_support(pci_dev))
916 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
917
918 if (!pm) {
919 pci_pm_default_suspend(pci_dev);
920 goto Fixup;
921 }
922
923 if (pm->poweroff) {
924 int error;
925
926 error = pm->poweroff(dev);
927 suspend_report_result(pm->poweroff, error);
928 if (error)
929 return error;
930 }
931
932 Fixup:
933 pci_fixup_device(pci_fixup_suspend, pci_dev);
934
935 return 0;
936}
937
938static int pci_pm_poweroff_noirq(struct device *dev)
939{
940 struct pci_dev *pci_dev = to_pci_dev(dev);
941 struct device_driver *drv = dev->driver;
942
943 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
944 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
945
946 if (!drv || !drv->pm)
947 return 0;
948
949 if (drv->pm->poweroff_noirq) {
950 int error;
951
952 error = drv->pm->poweroff_noirq(dev);
953 suspend_report_result(drv->pm->poweroff_noirq, error);
954 if (error)
955 return error;
956 }
957
958 if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
959 pci_prepare_to_sleep(pci_dev);
960
961 /*
962 * The reason for doing this here is the same as for the analogous code
963 * in pci_pm_suspend_noirq().
964 */
965 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
966 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
967
968 return 0;
969}
970
971static int pci_pm_restore_noirq(struct device *dev)
972{
973 struct pci_dev *pci_dev = to_pci_dev(dev);
974 struct device_driver *drv = dev->driver;
975 int error = 0;
976
977 pci_pm_default_resume_early(pci_dev);
978
979 if (pci_has_legacy_pm_support(pci_dev))
980 return pci_legacy_resume_early(dev);
981
982 if (drv && drv->pm && drv->pm->restore_noirq)
983 error = drv->pm->restore_noirq(dev);
984
985 return error;
986}
987
988static int pci_pm_restore(struct device *dev)
989{
990 struct pci_dev *pci_dev = to_pci_dev(dev);
991 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
992 int error = 0;
993
994 /*
995 * This is necessary for the hibernation error path in which restore is
996 * called without restoring the standard config registers of the device.
997 */
998 if (pci_dev->state_saved)
999 pci_restore_standard_config(pci_dev);
1000
1001 if (pci_has_legacy_pm_support(pci_dev))
1002 return pci_legacy_resume(dev);
1003
1004 pci_pm_default_resume(pci_dev);
1005
1006 if (pm) {
1007 if (pm->restore)
1008 error = pm->restore(dev);
1009 } else {
1010 pci_pm_reenable_device(pci_dev);
1011 }
1012
1013 return error;
1014}
1015
1016#else /* !CONFIG_HIBERNATE_CALLBACKS */
1017
1018#define pci_pm_freeze NULL
1019#define pci_pm_freeze_noirq NULL
1020#define pci_pm_thaw NULL
1021#define pci_pm_thaw_noirq NULL
1022#define pci_pm_poweroff NULL
1023#define pci_pm_poweroff_noirq NULL
1024#define pci_pm_restore NULL
1025#define pci_pm_restore_noirq NULL
1026
1027#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1028
1029#ifdef CONFIG_PM_RUNTIME
1030
1031static int pci_pm_runtime_suspend(struct device *dev)
1032{
1033 struct pci_dev *pci_dev = to_pci_dev(dev);
1034 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1035 pci_power_t prev = pci_dev->current_state;
1036 int error;
1037
1038 if (!pm || !pm->runtime_suspend)
1039 return -ENOSYS;
1040
1041 error = pm->runtime_suspend(dev);
1042 suspend_report_result(pm->runtime_suspend, error);
1043 if (error)
1044 return error;
1045
1046 pci_fixup_device(pci_fixup_suspend, pci_dev);
1047
1048 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1049 && pci_dev->current_state != PCI_UNKNOWN) {
1050 WARN_ONCE(pci_dev->current_state != prev,
1051 "PCI PM: State of device not saved by %pF\n",
1052 pm->runtime_suspend);
1053 return 0;
1054 }
1055
1056 if (!pci_dev->state_saved)
1057 pci_save_state(pci_dev);
1058
1059 pci_finish_runtime_suspend(pci_dev);
1060
1061 return 0;
1062}
1063
1064static int pci_pm_runtime_resume(struct device *dev)
1065{
1066 struct pci_dev *pci_dev = to_pci_dev(dev);
1067 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1068
1069 if (!pm || !pm->runtime_resume)
1070 return -ENOSYS;
1071
1072 pci_pm_default_resume_early(pci_dev);
1073 __pci_enable_wake(pci_dev, PCI_D0, true, false);
1074 pci_fixup_device(pci_fixup_resume, pci_dev);
1075
1076 return pm->runtime_resume(dev);
1077}
1078
1079static int pci_pm_runtime_idle(struct device *dev)
1080{
1081 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1082
1083 if (!pm)
1084 return -ENOSYS;
1085
1086 if (pm->runtime_idle) {
1087 int ret = pm->runtime_idle(dev);
1088 if (ret)
1089 return ret;
1090 }
1091
1092 pm_runtime_suspend(dev);
1093
1094 return 0;
1095}
1096
1097#else /* !CONFIG_PM_RUNTIME */
1098
1099#define pci_pm_runtime_suspend NULL
1100#define pci_pm_runtime_resume NULL
1101#define pci_pm_runtime_idle NULL
1102
1103#endif /* !CONFIG_PM_RUNTIME */
1104
1105#ifdef CONFIG_PM
1106
1107const struct dev_pm_ops pci_dev_pm_ops = {
1108 .prepare = pci_pm_prepare,
1109 .complete = pci_pm_complete,
1110 .suspend = pci_pm_suspend,
1111 .resume = pci_pm_resume,
1112 .freeze = pci_pm_freeze,
1113 .thaw = pci_pm_thaw,
1114 .poweroff = pci_pm_poweroff,
1115 .restore = pci_pm_restore,
1116 .suspend_noirq = pci_pm_suspend_noirq,
1117 .resume_noirq = pci_pm_resume_noirq,
1118 .freeze_noirq = pci_pm_freeze_noirq,
1119 .thaw_noirq = pci_pm_thaw_noirq,
1120 .poweroff_noirq = pci_pm_poweroff_noirq,
1121 .restore_noirq = pci_pm_restore_noirq,
1122 .runtime_suspend = pci_pm_runtime_suspend,
1123 .runtime_resume = pci_pm_runtime_resume,
1124 .runtime_idle = pci_pm_runtime_idle,
1125};
1126
1127#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1128
1129#else /* !COMFIG_PM_OPS */
1130
1131#define PCI_PM_OPS_PTR NULL
1132
1133#endif /* !COMFIG_PM_OPS */
1134
1135/**
1136 * __pci_register_driver - register a new pci driver
1137 * @drv: the driver structure to register
1138 * @owner: owner module of drv
1139 * @mod_name: module name string
1140 *
1141 * Adds the driver structure to the list of registered drivers.
1142 * Returns a negative value on error, otherwise 0.
1143 * If no error occurred, the driver remains registered even if
1144 * no device was claimed during registration.
1145 */
1146int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1147 const char *mod_name)
1148{
1149 int error;
1150
1151 /* initialize common driver fields */
1152 drv->driver.name = drv->name;
1153 drv->driver.bus = &pci_bus_type;
1154 drv->driver.owner = owner;
1155 drv->driver.mod_name = mod_name;
1156
1157 spin_lock_init(&drv->dynids.lock);
1158 INIT_LIST_HEAD(&drv->dynids.list);
1159
1160 /* register with core */
1161 error = driver_register(&drv->driver);
1162 if (error)
1163 goto out;
1164
1165 error = pci_create_newid_files(drv);
1166 if (error)
1167 goto out_newid;
1168out:
1169 return error;
1170
1171out_newid:
1172 driver_unregister(&drv->driver);
1173 goto out;
1174}
1175
1176/**
1177 * pci_unregister_driver - unregister a pci driver
1178 * @drv: the driver structure to unregister
1179 *
1180 * Deletes the driver structure from the list of registered PCI drivers,
1181 * gives it a chance to clean up by calling its remove() function for
1182 * each device it was responsible for, and marks those devices as
1183 * driverless.
1184 */
1185
1186void
1187pci_unregister_driver(struct pci_driver *drv)
1188{
1189 pci_remove_newid_files(drv);
1190 driver_unregister(&drv->driver);
1191 pci_free_dynids(drv);
1192}
1193
1194static struct pci_driver pci_compat_driver = {
1195 .name = "compat"
1196};
1197
1198/**
1199 * pci_dev_driver - get the pci_driver of a device
1200 * @dev: the device to query
1201 *
1202 * Returns the appropriate pci_driver structure or %NULL if there is no
1203 * registered driver for the device.
1204 */
1205struct pci_driver *
1206pci_dev_driver(const struct pci_dev *dev)
1207{
1208 if (dev->driver)
1209 return dev->driver;
1210 else {
1211 int i;
1212 for(i=0; i<=PCI_ROM_RESOURCE; i++)
1213 if (dev->resource[i].flags & IORESOURCE_BUSY)
1214 return &pci_compat_driver;
1215 }
1216 return NULL;
1217}
1218
1219/**
1220 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1221 * @dev: the PCI device structure to match against
1222 * @drv: the device driver to search for matching PCI device id structures
1223 *
1224 * Used by a driver to check whether a PCI device present in the
1225 * system is in its list of supported devices. Returns the matching
1226 * pci_device_id structure or %NULL if there is no match.
1227 */
1228static int pci_bus_match(struct device *dev, struct device_driver *drv)
1229{
1230 struct pci_dev *pci_dev = to_pci_dev(dev);
1231 struct pci_driver *pci_drv = to_pci_driver(drv);
1232 const struct pci_device_id *found_id;
1233
1234 found_id = pci_match_device(pci_drv, pci_dev);
1235 if (found_id)
1236 return 1;
1237
1238 return 0;
1239}
1240
1241/**
1242 * pci_dev_get - increments the reference count of the pci device structure
1243 * @dev: the device being referenced
1244 *
1245 * Each live reference to a device should be refcounted.
1246 *
1247 * Drivers for PCI devices should normally record such references in
1248 * their probe() methods, when they bind to a device, and release
1249 * them by calling pci_dev_put(), in their disconnect() methods.
1250 *
1251 * A pointer to the device with the incremented reference counter is returned.
1252 */
1253struct pci_dev *pci_dev_get(struct pci_dev *dev)
1254{
1255 if (dev)
1256 get_device(&dev->dev);
1257 return dev;
1258}
1259
1260/**
1261 * pci_dev_put - release a use of the pci device structure
1262 * @dev: device that's been disconnected
1263 *
1264 * Must be called when a user of a device is finished with it. When the last
1265 * user of the device calls this function, the memory of the device is freed.
1266 */
1267void pci_dev_put(struct pci_dev *dev)
1268{
1269 if (dev)
1270 put_device(&dev->dev);
1271}
1272
1273#ifndef CONFIG_HOTPLUG
1274int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1275{
1276 return -ENODEV;
1277}
1278#endif
1279
1280struct bus_type pci_bus_type = {
1281 .name = "pci",
1282 .match = pci_bus_match,
1283 .uevent = pci_uevent,
1284 .probe = pci_device_probe,
1285 .remove = pci_device_remove,
1286 .shutdown = pci_device_shutdown,
1287 .dev_attrs = pci_dev_attrs,
1288 .bus_attrs = pci_bus_attrs,
1289 .pm = PCI_PM_OPS_PTR,
1290};
1291
1292static int __init pci_driver_init(void)
1293{
1294 return bus_register(&pci_bus_type);
1295}
1296
1297postcore_initcall(pci_driver_init);
1298
1299EXPORT_SYMBOL_GPL(pci_add_dynid);
1300EXPORT_SYMBOL(pci_match_id);
1301EXPORT_SYMBOL(__pci_register_driver);
1302EXPORT_SYMBOL(pci_unregister_driver);
1303EXPORT_SYMBOL(pci_dev_driver);
1304EXPORT_SYMBOL(pci_bus_type);
1305EXPORT_SYMBOL(pci_dev_get);
1306EXPORT_SYMBOL(pci_dev_put);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
4 * (C) Copyright 2007 Novell Inc.
5 */
6
7#include <linux/pci.h>
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/device.h>
11#include <linux/mempolicy.h>
12#include <linux/string.h>
13#include <linux/slab.h>
14#include <linux/sched.h>
15#include <linux/cpu.h>
16#include <linux/pm_runtime.h>
17#include <linux/suspend.h>
18#include <linux/kexec.h>
19#include "pci.h"
20#include "pcie/portdrv.h"
21
22struct pci_dynid {
23 struct list_head node;
24 struct pci_device_id id;
25};
26
27/**
28 * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
29 * @drv: target pci driver
30 * @vendor: PCI vendor ID
31 * @device: PCI device ID
32 * @subvendor: PCI subvendor ID
33 * @subdevice: PCI subdevice ID
34 * @class: PCI class
35 * @class_mask: PCI class mask
36 * @driver_data: private driver data
37 *
38 * Adds a new dynamic pci device ID to this driver and causes the
39 * driver to probe for all devices again. @drv must have been
40 * registered prior to calling this function.
41 *
42 * CONTEXT:
43 * Does GFP_KERNEL allocation.
44 *
45 * RETURNS:
46 * 0 on success, -errno on failure.
47 */
48int pci_add_dynid(struct pci_driver *drv,
49 unsigned int vendor, unsigned int device,
50 unsigned int subvendor, unsigned int subdevice,
51 unsigned int class, unsigned int class_mask,
52 unsigned long driver_data)
53{
54 struct pci_dynid *dynid;
55
56 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
57 if (!dynid)
58 return -ENOMEM;
59
60 dynid->id.vendor = vendor;
61 dynid->id.device = device;
62 dynid->id.subvendor = subvendor;
63 dynid->id.subdevice = subdevice;
64 dynid->id.class = class;
65 dynid->id.class_mask = class_mask;
66 dynid->id.driver_data = driver_data;
67
68 spin_lock(&drv->dynids.lock);
69 list_add_tail(&dynid->node, &drv->dynids.list);
70 spin_unlock(&drv->dynids.lock);
71
72 return driver_attach(&drv->driver);
73}
74EXPORT_SYMBOL_GPL(pci_add_dynid);
75
76static void pci_free_dynids(struct pci_driver *drv)
77{
78 struct pci_dynid *dynid, *n;
79
80 spin_lock(&drv->dynids.lock);
81 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
82 list_del(&dynid->node);
83 kfree(dynid);
84 }
85 spin_unlock(&drv->dynids.lock);
86}
87
88/**
89 * store_new_id - sysfs frontend to pci_add_dynid()
90 * @driver: target device driver
91 * @buf: buffer for scanning device ID data
92 * @count: input size
93 *
94 * Allow PCI IDs to be added to an existing driver via sysfs.
95 */
96static ssize_t new_id_store(struct device_driver *driver, const char *buf,
97 size_t count)
98{
99 struct pci_driver *pdrv = to_pci_driver(driver);
100 const struct pci_device_id *ids = pdrv->id_table;
101 __u32 vendor, device, subvendor = PCI_ANY_ID,
102 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
103 unsigned long driver_data = 0;
104 int fields = 0;
105 int retval = 0;
106
107 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
108 &vendor, &device, &subvendor, &subdevice,
109 &class, &class_mask, &driver_data);
110 if (fields < 2)
111 return -EINVAL;
112
113 if (fields != 7) {
114 struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
115 if (!pdev)
116 return -ENOMEM;
117
118 pdev->vendor = vendor;
119 pdev->device = device;
120 pdev->subsystem_vendor = subvendor;
121 pdev->subsystem_device = subdevice;
122 pdev->class = class;
123
124 if (pci_match_id(pdrv->id_table, pdev))
125 retval = -EEXIST;
126
127 kfree(pdev);
128
129 if (retval)
130 return retval;
131 }
132
133 /* Only accept driver_data values that match an existing id_table
134 entry */
135 if (ids) {
136 retval = -EINVAL;
137 while (ids->vendor || ids->subvendor || ids->class_mask) {
138 if (driver_data == ids->driver_data) {
139 retval = 0;
140 break;
141 }
142 ids++;
143 }
144 if (retval) /* No match */
145 return retval;
146 }
147
148 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
149 class, class_mask, driver_data);
150 if (retval)
151 return retval;
152 return count;
153}
154static DRIVER_ATTR_WO(new_id);
155
156/**
157 * store_remove_id - remove a PCI device ID from this driver
158 * @driver: target device driver
159 * @buf: buffer for scanning device ID data
160 * @count: input size
161 *
162 * Removes a dynamic pci device ID to this driver.
163 */
164static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
165 size_t count)
166{
167 struct pci_dynid *dynid, *n;
168 struct pci_driver *pdrv = to_pci_driver(driver);
169 __u32 vendor, device, subvendor = PCI_ANY_ID,
170 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
171 int fields = 0;
172 size_t retval = -ENODEV;
173
174 fields = sscanf(buf, "%x %x %x %x %x %x",
175 &vendor, &device, &subvendor, &subdevice,
176 &class, &class_mask);
177 if (fields < 2)
178 return -EINVAL;
179
180 spin_lock(&pdrv->dynids.lock);
181 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
182 struct pci_device_id *id = &dynid->id;
183 if ((id->vendor == vendor) &&
184 (id->device == device) &&
185 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
186 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
187 !((id->class ^ class) & class_mask)) {
188 list_del(&dynid->node);
189 kfree(dynid);
190 retval = count;
191 break;
192 }
193 }
194 spin_unlock(&pdrv->dynids.lock);
195
196 return retval;
197}
198static DRIVER_ATTR_WO(remove_id);
199
200static struct attribute *pci_drv_attrs[] = {
201 &driver_attr_new_id.attr,
202 &driver_attr_remove_id.attr,
203 NULL,
204};
205ATTRIBUTE_GROUPS(pci_drv);
206
207/**
208 * pci_match_id - See if a pci device matches a given pci_id table
209 * @ids: array of PCI device id structures to search in
210 * @dev: the PCI device structure to match against.
211 *
212 * Used by a driver to check whether a PCI device present in the
213 * system is in its list of supported devices. Returns the matching
214 * pci_device_id structure or %NULL if there is no match.
215 *
216 * Deprecated, don't use this as it will not catch any dynamic ids
217 * that a driver might want to check for.
218 */
219const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
220 struct pci_dev *dev)
221{
222 if (ids) {
223 while (ids->vendor || ids->subvendor || ids->class_mask) {
224 if (pci_match_one_device(ids, dev))
225 return ids;
226 ids++;
227 }
228 }
229 return NULL;
230}
231EXPORT_SYMBOL(pci_match_id);
232
233static const struct pci_device_id pci_device_id_any = {
234 .vendor = PCI_ANY_ID,
235 .device = PCI_ANY_ID,
236 .subvendor = PCI_ANY_ID,
237 .subdevice = PCI_ANY_ID,
238};
239
240/**
241 * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
242 * @drv: the PCI driver to match against
243 * @dev: the PCI device structure to match against
244 *
245 * Used by a driver to check whether a PCI device present in the
246 * system is in its list of supported devices. Returns the matching
247 * pci_device_id structure or %NULL if there is no match.
248 */
249static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
250 struct pci_dev *dev)
251{
252 struct pci_dynid *dynid;
253 const struct pci_device_id *found_id = NULL;
254
255 /* When driver_override is set, only bind to the matching driver */
256 if (dev->driver_override && strcmp(dev->driver_override, drv->name))
257 return NULL;
258
259 /* Look at the dynamic ids first, before the static ones */
260 spin_lock(&drv->dynids.lock);
261 list_for_each_entry(dynid, &drv->dynids.list, node) {
262 if (pci_match_one_device(&dynid->id, dev)) {
263 found_id = &dynid->id;
264 break;
265 }
266 }
267 spin_unlock(&drv->dynids.lock);
268
269 if (!found_id)
270 found_id = pci_match_id(drv->id_table, dev);
271
272 /* driver_override will always match, send a dummy id */
273 if (!found_id && dev->driver_override)
274 found_id = &pci_device_id_any;
275
276 return found_id;
277}
278
279struct drv_dev_and_id {
280 struct pci_driver *drv;
281 struct pci_dev *dev;
282 const struct pci_device_id *id;
283};
284
285static long local_pci_probe(void *_ddi)
286{
287 struct drv_dev_and_id *ddi = _ddi;
288 struct pci_dev *pci_dev = ddi->dev;
289 struct pci_driver *pci_drv = ddi->drv;
290 struct device *dev = &pci_dev->dev;
291 int rc;
292
293 /*
294 * Unbound PCI devices are always put in D0, regardless of
295 * runtime PM status. During probe, the device is set to
296 * active and the usage count is incremented. If the driver
297 * supports runtime PM, it should call pm_runtime_put_noidle(),
298 * or any other runtime PM helper function decrementing the usage
299 * count, in its probe routine and pm_runtime_get_noresume() in
300 * its remove routine.
301 */
302 pm_runtime_get_sync(dev);
303 pci_dev->driver = pci_drv;
304 rc = pci_drv->probe(pci_dev, ddi->id);
305 if (!rc)
306 return rc;
307 if (rc < 0) {
308 pci_dev->driver = NULL;
309 pm_runtime_put_sync(dev);
310 return rc;
311 }
312 /*
313 * Probe function should return < 0 for failure, 0 for success
314 * Treat values > 0 as success, but warn.
315 */
316 dev_warn(dev, "Driver probe function unexpectedly returned %d\n", rc);
317 return 0;
318}
319
320static bool pci_physfn_is_probed(struct pci_dev *dev)
321{
322#ifdef CONFIG_PCI_IOV
323 return dev->is_virtfn && dev->physfn->is_probed;
324#else
325 return false;
326#endif
327}
328
329static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
330 const struct pci_device_id *id)
331{
332 int error, node, cpu;
333 struct drv_dev_and_id ddi = { drv, dev, id };
334
335 /*
336 * Execute driver initialization on node where the device is
337 * attached. This way the driver likely allocates its local memory
338 * on the right node.
339 */
340 node = dev_to_node(&dev->dev);
341 dev->is_probed = 1;
342
343 cpu_hotplug_disable();
344
345 /*
346 * Prevent nesting work_on_cpu() for the case where a Virtual Function
347 * device is probed from work_on_cpu() of the Physical device.
348 */
349 if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
350 pci_physfn_is_probed(dev))
351 cpu = nr_cpu_ids;
352 else
353 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
354
355 if (cpu < nr_cpu_ids)
356 error = work_on_cpu(cpu, local_pci_probe, &ddi);
357 else
358 error = local_pci_probe(&ddi);
359
360 dev->is_probed = 0;
361 cpu_hotplug_enable();
362 return error;
363}
364
365/**
366 * __pci_device_probe - check if a driver wants to claim a specific PCI device
367 * @drv: driver to call to check if it wants the PCI device
368 * @pci_dev: PCI device being probed
369 *
370 * returns 0 on success, else error.
371 * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
372 */
373static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
374{
375 const struct pci_device_id *id;
376 int error = 0;
377
378 if (!pci_dev->driver && drv->probe) {
379 error = -ENODEV;
380
381 id = pci_match_device(drv, pci_dev);
382 if (id)
383 error = pci_call_probe(drv, pci_dev, id);
384 }
385 return error;
386}
387
388int __weak pcibios_alloc_irq(struct pci_dev *dev)
389{
390 return 0;
391}
392
393void __weak pcibios_free_irq(struct pci_dev *dev)
394{
395}
396
397#ifdef CONFIG_PCI_IOV
398static inline bool pci_device_can_probe(struct pci_dev *pdev)
399{
400 return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe);
401}
402#else
403static inline bool pci_device_can_probe(struct pci_dev *pdev)
404{
405 return true;
406}
407#endif
408
409static int pci_device_probe(struct device *dev)
410{
411 int error;
412 struct pci_dev *pci_dev = to_pci_dev(dev);
413 struct pci_driver *drv = to_pci_driver(dev->driver);
414
415 pci_assign_irq(pci_dev);
416
417 error = pcibios_alloc_irq(pci_dev);
418 if (error < 0)
419 return error;
420
421 pci_dev_get(pci_dev);
422 if (pci_device_can_probe(pci_dev)) {
423 error = __pci_device_probe(drv, pci_dev);
424 if (error) {
425 pcibios_free_irq(pci_dev);
426 pci_dev_put(pci_dev);
427 }
428 }
429
430 return error;
431}
432
433static int pci_device_remove(struct device *dev)
434{
435 struct pci_dev *pci_dev = to_pci_dev(dev);
436 struct pci_driver *drv = pci_dev->driver;
437
438 if (drv) {
439 if (drv->remove) {
440 pm_runtime_get_sync(dev);
441 drv->remove(pci_dev);
442 pm_runtime_put_noidle(dev);
443 }
444 pcibios_free_irq(pci_dev);
445 pci_dev->driver = NULL;
446 }
447
448 /* Undo the runtime PM settings in local_pci_probe() */
449 pm_runtime_put_sync(dev);
450
451 /*
452 * If the device is still on, set the power state as "unknown",
453 * since it might change by the next time we load the driver.
454 */
455 if (pci_dev->current_state == PCI_D0)
456 pci_dev->current_state = PCI_UNKNOWN;
457
458 /*
459 * We would love to complain here if pci_dev->is_enabled is set, that
460 * the driver should have called pci_disable_device(), but the
461 * unfortunate fact is there are too many odd BIOS and bridge setups
462 * that don't like drivers doing that all of the time.
463 * Oh well, we can dream of sane hardware when we sleep, no matter how
464 * horrible the crap we have to deal with is when we are awake...
465 */
466
467 pci_dev_put(pci_dev);
468 return 0;
469}
470
471static void pci_device_shutdown(struct device *dev)
472{
473 struct pci_dev *pci_dev = to_pci_dev(dev);
474 struct pci_driver *drv = pci_dev->driver;
475
476 pm_runtime_resume(dev);
477
478 if (drv && drv->shutdown)
479 drv->shutdown(pci_dev);
480
481 /*
482 * If this is a kexec reboot, turn off Bus Master bit on the
483 * device to tell it to not continue to do DMA. Don't touch
484 * devices in D3cold or unknown states.
485 * If it is not a kexec reboot, firmware will hit the PCI
486 * devices with big hammer and stop their DMA any way.
487 */
488 if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
489 pci_clear_master(pci_dev);
490}
491
492#ifdef CONFIG_PM
493
494/* Auxiliary functions used for system resume and run-time resume. */
495
496/**
497 * pci_restore_standard_config - restore standard config registers of PCI device
498 * @pci_dev: PCI device to handle
499 */
500static int pci_restore_standard_config(struct pci_dev *pci_dev)
501{
502 pci_update_current_state(pci_dev, PCI_UNKNOWN);
503
504 if (pci_dev->current_state != PCI_D0) {
505 int error = pci_set_power_state(pci_dev, PCI_D0);
506 if (error)
507 return error;
508 }
509
510 pci_restore_state(pci_dev);
511 pci_pme_restore(pci_dev);
512 return 0;
513}
514
515#endif
516
517#ifdef CONFIG_PM_SLEEP
518
519static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
520{
521 pci_power_up(pci_dev);
522 pci_restore_state(pci_dev);
523 pci_pme_restore(pci_dev);
524 pci_fixup_device(pci_fixup_resume_early, pci_dev);
525}
526
527/*
528 * Default "suspend" method for devices that have no driver provided suspend,
529 * or not even a driver at all (second part).
530 */
531static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
532{
533 /*
534 * mark its power state as "unknown", since we don't know if
535 * e.g. the BIOS will change its device state when we suspend.
536 */
537 if (pci_dev->current_state == PCI_D0)
538 pci_dev->current_state = PCI_UNKNOWN;
539}
540
541/*
542 * Default "resume" method for devices that have no driver provided resume,
543 * or not even a driver at all (second part).
544 */
545static int pci_pm_reenable_device(struct pci_dev *pci_dev)
546{
547 int retval;
548
549 /* if the device was enabled before suspend, reenable */
550 retval = pci_reenable_device(pci_dev);
551 /*
552 * if the device was busmaster before the suspend, make it busmaster
553 * again
554 */
555 if (pci_dev->is_busmaster)
556 pci_set_master(pci_dev);
557
558 return retval;
559}
560
561static int pci_legacy_suspend(struct device *dev, pm_message_t state)
562{
563 struct pci_dev *pci_dev = to_pci_dev(dev);
564 struct pci_driver *drv = pci_dev->driver;
565
566 if (drv && drv->suspend) {
567 pci_power_t prev = pci_dev->current_state;
568 int error;
569
570 error = drv->suspend(pci_dev, state);
571 suspend_report_result(drv->suspend, error);
572 if (error)
573 return error;
574
575 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
576 && pci_dev->current_state != PCI_UNKNOWN) {
577 WARN_ONCE(pci_dev->current_state != prev,
578 "PCI PM: Device state not saved by %pF\n",
579 drv->suspend);
580 }
581 }
582
583 pci_fixup_device(pci_fixup_suspend, pci_dev);
584
585 return 0;
586}
587
588static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
589{
590 struct pci_dev *pci_dev = to_pci_dev(dev);
591 struct pci_driver *drv = pci_dev->driver;
592
593 if (drv && drv->suspend_late) {
594 pci_power_t prev = pci_dev->current_state;
595 int error;
596
597 error = drv->suspend_late(pci_dev, state);
598 suspend_report_result(drv->suspend_late, error);
599 if (error)
600 return error;
601
602 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
603 && pci_dev->current_state != PCI_UNKNOWN) {
604 WARN_ONCE(pci_dev->current_state != prev,
605 "PCI PM: Device state not saved by %pF\n",
606 drv->suspend_late);
607 goto Fixup;
608 }
609 }
610
611 if (!pci_dev->state_saved)
612 pci_save_state(pci_dev);
613
614 pci_pm_set_unknown_state(pci_dev);
615
616Fixup:
617 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
618
619 return 0;
620}
621
622static int pci_legacy_resume_early(struct device *dev)
623{
624 struct pci_dev *pci_dev = to_pci_dev(dev);
625 struct pci_driver *drv = pci_dev->driver;
626
627 return drv && drv->resume_early ?
628 drv->resume_early(pci_dev) : 0;
629}
630
631static int pci_legacy_resume(struct device *dev)
632{
633 struct pci_dev *pci_dev = to_pci_dev(dev);
634 struct pci_driver *drv = pci_dev->driver;
635
636 pci_fixup_device(pci_fixup_resume, pci_dev);
637
638 return drv && drv->resume ?
639 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
640}
641
642/* Auxiliary functions used by the new power management framework */
643
644static void pci_pm_default_resume(struct pci_dev *pci_dev)
645{
646 pci_fixup_device(pci_fixup_resume, pci_dev);
647 pci_enable_wake(pci_dev, PCI_D0, false);
648}
649
650static void pci_pm_default_suspend(struct pci_dev *pci_dev)
651{
652 /* Disable non-bridge devices without PM support */
653 if (!pci_has_subordinate(pci_dev))
654 pci_disable_enabled_device(pci_dev);
655}
656
657static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
658{
659 struct pci_driver *drv = pci_dev->driver;
660 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
661 || drv->resume_early);
662
663 /*
664 * Legacy PM support is used by default, so warn if the new framework is
665 * supported as well. Drivers are supposed to support either the
666 * former, or the latter, but not both at the same time.
667 */
668 WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
669 drv->name, pci_dev->vendor, pci_dev->device);
670
671 return ret;
672}
673
674/* New power management framework */
675
676static int pci_pm_prepare(struct device *dev)
677{
678 struct device_driver *drv = dev->driver;
679
680 if (drv && drv->pm && drv->pm->prepare) {
681 int error = drv->pm->prepare(dev);
682 if (error < 0)
683 return error;
684
685 if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
686 return 0;
687 }
688 return pci_dev_keep_suspended(to_pci_dev(dev));
689}
690
691static void pci_pm_complete(struct device *dev)
692{
693 struct pci_dev *pci_dev = to_pci_dev(dev);
694
695 pci_dev_complete_resume(pci_dev);
696 pm_generic_complete(dev);
697
698 /* Resume device if platform firmware has put it in reset-power-on */
699 if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
700 pci_power_t pre_sleep_state = pci_dev->current_state;
701
702 pci_update_current_state(pci_dev, pci_dev->current_state);
703 if (pci_dev->current_state < pre_sleep_state)
704 pm_request_resume(dev);
705 }
706}
707
708#else /* !CONFIG_PM_SLEEP */
709
710#define pci_pm_prepare NULL
711#define pci_pm_complete NULL
712
713#endif /* !CONFIG_PM_SLEEP */
714
715#ifdef CONFIG_SUSPEND
716static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
717{
718 /*
719 * Some BIOSes forget to clear Root PME Status bits after system
720 * wakeup, which breaks ACPI-based runtime wakeup on PCI Express.
721 * Clear those bits now just in case (shouldn't hurt).
722 */
723 if (pci_is_pcie(pci_dev) &&
724 (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
725 pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
726 pcie_clear_root_pme_status(pci_dev);
727}
728
729static int pci_pm_suspend(struct device *dev)
730{
731 struct pci_dev *pci_dev = to_pci_dev(dev);
732 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
733
734 if (pci_has_legacy_pm_support(pci_dev))
735 return pci_legacy_suspend(dev, PMSG_SUSPEND);
736
737 if (!pm) {
738 pci_pm_default_suspend(pci_dev);
739 return 0;
740 }
741
742 /*
743 * PCI devices suspended at run time may need to be resumed at this
744 * point, because in general it may be necessary to reconfigure them for
745 * system suspend. Namely, if the device is expected to wake up the
746 * system from the sleep state, it may have to be reconfigured for this
747 * purpose, or if the device is not expected to wake up the system from
748 * the sleep state, it should be prevented from signaling wakeup events
749 * going forward.
750 *
751 * Also if the driver of the device does not indicate that its system
752 * suspend callbacks can cope with runtime-suspended devices, it is
753 * better to resume the device from runtime suspend here.
754 */
755 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
756 !pci_dev_keep_suspended(pci_dev))
757 pm_runtime_resume(dev);
758
759 pci_dev->state_saved = false;
760 if (pm->suspend) {
761 pci_power_t prev = pci_dev->current_state;
762 int error;
763
764 error = pm->suspend(dev);
765 suspend_report_result(pm->suspend, error);
766 if (error)
767 return error;
768
769 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
770 && pci_dev->current_state != PCI_UNKNOWN) {
771 WARN_ONCE(pci_dev->current_state != prev,
772 "PCI PM: State of device not saved by %pF\n",
773 pm->suspend);
774 }
775 }
776
777 return 0;
778}
779
780static int pci_pm_suspend_late(struct device *dev)
781{
782 if (dev_pm_smart_suspend_and_suspended(dev))
783 return 0;
784
785 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
786
787 return pm_generic_suspend_late(dev);
788}
789
790static int pci_pm_suspend_noirq(struct device *dev)
791{
792 struct pci_dev *pci_dev = to_pci_dev(dev);
793 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
794
795 if (dev_pm_smart_suspend_and_suspended(dev)) {
796 dev->power.may_skip_resume = true;
797 return 0;
798 }
799
800 if (pci_has_legacy_pm_support(pci_dev))
801 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
802
803 if (!pm) {
804 pci_save_state(pci_dev);
805 goto Fixup;
806 }
807
808 if (pm->suspend_noirq) {
809 pci_power_t prev = pci_dev->current_state;
810 int error;
811
812 error = pm->suspend_noirq(dev);
813 suspend_report_result(pm->suspend_noirq, error);
814 if (error)
815 return error;
816
817 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
818 && pci_dev->current_state != PCI_UNKNOWN) {
819 WARN_ONCE(pci_dev->current_state != prev,
820 "PCI PM: State of device not saved by %pF\n",
821 pm->suspend_noirq);
822 goto Fixup;
823 }
824 }
825
826 if (!pci_dev->state_saved) {
827 pci_save_state(pci_dev);
828 if (pci_power_manageable(pci_dev))
829 pci_prepare_to_sleep(pci_dev);
830 }
831
832 dev_dbg(dev, "PCI PM: Suspend power state: %s\n",
833 pci_power_name(pci_dev->current_state));
834
835 pci_pm_set_unknown_state(pci_dev);
836
837 /*
838 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
839 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
840 * hasn't been quiesced and tries to turn it off. If the controller
841 * is already in D3, this can hang or cause memory corruption.
842 *
843 * Since the value of the COMMAND register doesn't matter once the
844 * device has been suspended, we can safely set it to 0 here.
845 */
846 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
847 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
848
849Fixup:
850 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
851
852 /*
853 * If the target system sleep state is suspend-to-idle, it is sufficient
854 * to check whether or not the device's wakeup settings are good for
855 * runtime PM. Otherwise, the pm_resume_via_firmware() check will cause
856 * pci_pm_complete() to take care of fixing up the device's state
857 * anyway, if need be.
858 */
859 dev->power.may_skip_resume = device_may_wakeup(dev) ||
860 !device_can_wakeup(dev);
861
862 return 0;
863}
864
865static int pci_pm_resume_noirq(struct device *dev)
866{
867 struct pci_dev *pci_dev = to_pci_dev(dev);
868 struct device_driver *drv = dev->driver;
869 int error = 0;
870
871 if (dev_pm_may_skip_resume(dev))
872 return 0;
873
874 /*
875 * Devices with DPM_FLAG_SMART_SUSPEND may be left in runtime suspend
876 * during system suspend, so update their runtime PM status to "active"
877 * as they are going to be put into D0 shortly.
878 */
879 if (dev_pm_smart_suspend_and_suspended(dev))
880 pm_runtime_set_active(dev);
881
882 pci_pm_default_resume_early(pci_dev);
883
884 if (pci_has_legacy_pm_support(pci_dev))
885 return pci_legacy_resume_early(dev);
886
887 pcie_pme_root_status_cleanup(pci_dev);
888
889 if (drv && drv->pm && drv->pm->resume_noirq)
890 error = drv->pm->resume_noirq(dev);
891
892 return error;
893}
894
895static int pci_pm_resume(struct device *dev)
896{
897 struct pci_dev *pci_dev = to_pci_dev(dev);
898 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
899 int error = 0;
900
901 /*
902 * This is necessary for the suspend error path in which resume is
903 * called without restoring the standard config registers of the device.
904 */
905 if (pci_dev->state_saved)
906 pci_restore_standard_config(pci_dev);
907
908 if (pci_has_legacy_pm_support(pci_dev))
909 return pci_legacy_resume(dev);
910
911 pci_pm_default_resume(pci_dev);
912
913 if (pm) {
914 if (pm->resume)
915 error = pm->resume(dev);
916 } else {
917 pci_pm_reenable_device(pci_dev);
918 }
919
920 return error;
921}
922
923#else /* !CONFIG_SUSPEND */
924
925#define pci_pm_suspend NULL
926#define pci_pm_suspend_late NULL
927#define pci_pm_suspend_noirq NULL
928#define pci_pm_resume NULL
929#define pci_pm_resume_noirq NULL
930
931#endif /* !CONFIG_SUSPEND */
932
933#ifdef CONFIG_HIBERNATE_CALLBACKS
934
935
936/*
937 * pcibios_pm_ops - provide arch-specific hooks when a PCI device is doing
938 * a hibernate transition
939 */
940struct dev_pm_ops __weak pcibios_pm_ops;
941
942static int pci_pm_freeze(struct device *dev)
943{
944 struct pci_dev *pci_dev = to_pci_dev(dev);
945 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
946
947 if (pci_has_legacy_pm_support(pci_dev))
948 return pci_legacy_suspend(dev, PMSG_FREEZE);
949
950 if (!pm) {
951 pci_pm_default_suspend(pci_dev);
952 return 0;
953 }
954
955 /*
956 * This used to be done in pci_pm_prepare() for all devices and some
957 * drivers may depend on it, so do it here. Ideally, runtime-suspended
958 * devices should not be touched during freeze/thaw transitions,
959 * however.
960 */
961 if (!dev_pm_smart_suspend_and_suspended(dev)) {
962 pm_runtime_resume(dev);
963 pci_dev->state_saved = false;
964 }
965
966 if (pm->freeze) {
967 int error;
968
969 error = pm->freeze(dev);
970 suspend_report_result(pm->freeze, error);
971 if (error)
972 return error;
973 }
974
975 return 0;
976}
977
978static int pci_pm_freeze_late(struct device *dev)
979{
980 if (dev_pm_smart_suspend_and_suspended(dev))
981 return 0;
982
983 return pm_generic_freeze_late(dev);
984}
985
986static int pci_pm_freeze_noirq(struct device *dev)
987{
988 struct pci_dev *pci_dev = to_pci_dev(dev);
989 struct device_driver *drv = dev->driver;
990
991 if (dev_pm_smart_suspend_and_suspended(dev))
992 return 0;
993
994 if (pci_has_legacy_pm_support(pci_dev))
995 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
996
997 if (drv && drv->pm && drv->pm->freeze_noirq) {
998 int error;
999
1000 error = drv->pm->freeze_noirq(dev);
1001 suspend_report_result(drv->pm->freeze_noirq, error);
1002 if (error)
1003 return error;
1004 }
1005
1006 if (!pci_dev->state_saved)
1007 pci_save_state(pci_dev);
1008
1009 pci_pm_set_unknown_state(pci_dev);
1010
1011 if (pcibios_pm_ops.freeze_noirq)
1012 return pcibios_pm_ops.freeze_noirq(dev);
1013
1014 return 0;
1015}
1016
1017static int pci_pm_thaw_noirq(struct device *dev)
1018{
1019 struct pci_dev *pci_dev = to_pci_dev(dev);
1020 struct device_driver *drv = dev->driver;
1021 int error = 0;
1022
1023 /*
1024 * If the device is in runtime suspend, the code below may not work
1025 * correctly with it, so skip that code and make the PM core skip all of
1026 * the subsequent "thaw" callbacks for the device.
1027 */
1028 if (dev_pm_smart_suspend_and_suspended(dev)) {
1029 dev_pm_skip_next_resume_phases(dev);
1030 return 0;
1031 }
1032
1033 if (pcibios_pm_ops.thaw_noirq) {
1034 error = pcibios_pm_ops.thaw_noirq(dev);
1035 if (error)
1036 return error;
1037 }
1038
1039 if (pci_has_legacy_pm_support(pci_dev))
1040 return pci_legacy_resume_early(dev);
1041
1042 /*
1043 * pci_restore_state() requires the device to be in D0 (because of MSI
1044 * restoration among other things), so force it into D0 in case the
1045 * driver's "freeze" callbacks put it into a low-power state directly.
1046 */
1047 pci_set_power_state(pci_dev, PCI_D0);
1048 pci_restore_state(pci_dev);
1049
1050 if (drv && drv->pm && drv->pm->thaw_noirq)
1051 error = drv->pm->thaw_noirq(dev);
1052
1053 return error;
1054}
1055
1056static int pci_pm_thaw(struct device *dev)
1057{
1058 struct pci_dev *pci_dev = to_pci_dev(dev);
1059 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1060 int error = 0;
1061
1062 if (pci_has_legacy_pm_support(pci_dev))
1063 return pci_legacy_resume(dev);
1064
1065 if (pm) {
1066 if (pm->thaw)
1067 error = pm->thaw(dev);
1068 } else {
1069 pci_pm_reenable_device(pci_dev);
1070 }
1071
1072 pci_dev->state_saved = false;
1073
1074 return error;
1075}
1076
1077static int pci_pm_poweroff(struct device *dev)
1078{
1079 struct pci_dev *pci_dev = to_pci_dev(dev);
1080 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1081
1082 if (pci_has_legacy_pm_support(pci_dev))
1083 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1084
1085 if (!pm) {
1086 pci_pm_default_suspend(pci_dev);
1087 return 0;
1088 }
1089
1090 /* The reason to do that is the same as in pci_pm_suspend(). */
1091 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1092 !pci_dev_keep_suspended(pci_dev))
1093 pm_runtime_resume(dev);
1094
1095 pci_dev->state_saved = false;
1096 if (pm->poweroff) {
1097 int error;
1098
1099 error = pm->poweroff(dev);
1100 suspend_report_result(pm->poweroff, error);
1101 if (error)
1102 return error;
1103 }
1104
1105 return 0;
1106}
1107
1108static int pci_pm_poweroff_late(struct device *dev)
1109{
1110 if (dev_pm_smart_suspend_and_suspended(dev))
1111 return 0;
1112
1113 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
1114
1115 return pm_generic_poweroff_late(dev);
1116}
1117
1118static int pci_pm_poweroff_noirq(struct device *dev)
1119{
1120 struct pci_dev *pci_dev = to_pci_dev(dev);
1121 struct device_driver *drv = dev->driver;
1122
1123 if (dev_pm_smart_suspend_and_suspended(dev))
1124 return 0;
1125
1126 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
1127 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
1128
1129 if (!drv || !drv->pm) {
1130 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1131 return 0;
1132 }
1133
1134 if (drv->pm->poweroff_noirq) {
1135 int error;
1136
1137 error = drv->pm->poweroff_noirq(dev);
1138 suspend_report_result(drv->pm->poweroff_noirq, error);
1139 if (error)
1140 return error;
1141 }
1142
1143 if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1144 pci_prepare_to_sleep(pci_dev);
1145
1146 /*
1147 * The reason for doing this here is the same as for the analogous code
1148 * in pci_pm_suspend_noirq().
1149 */
1150 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1151 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1152
1153 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1154
1155 if (pcibios_pm_ops.poweroff_noirq)
1156 return pcibios_pm_ops.poweroff_noirq(dev);
1157
1158 return 0;
1159}
1160
1161static int pci_pm_restore_noirq(struct device *dev)
1162{
1163 struct pci_dev *pci_dev = to_pci_dev(dev);
1164 struct device_driver *drv = dev->driver;
1165 int error = 0;
1166
1167 /* This is analogous to the pci_pm_resume_noirq() case. */
1168 if (dev_pm_smart_suspend_and_suspended(dev))
1169 pm_runtime_set_active(dev);
1170
1171 if (pcibios_pm_ops.restore_noirq) {
1172 error = pcibios_pm_ops.restore_noirq(dev);
1173 if (error)
1174 return error;
1175 }
1176
1177 pci_pm_default_resume_early(pci_dev);
1178
1179 if (pci_has_legacy_pm_support(pci_dev))
1180 return pci_legacy_resume_early(dev);
1181
1182 if (drv && drv->pm && drv->pm->restore_noirq)
1183 error = drv->pm->restore_noirq(dev);
1184
1185 return error;
1186}
1187
1188static int pci_pm_restore(struct device *dev)
1189{
1190 struct pci_dev *pci_dev = to_pci_dev(dev);
1191 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1192 int error = 0;
1193
1194 /*
1195 * This is necessary for the hibernation error path in which restore is
1196 * called without restoring the standard config registers of the device.
1197 */
1198 if (pci_dev->state_saved)
1199 pci_restore_standard_config(pci_dev);
1200
1201 if (pci_has_legacy_pm_support(pci_dev))
1202 return pci_legacy_resume(dev);
1203
1204 pci_pm_default_resume(pci_dev);
1205
1206 if (pm) {
1207 if (pm->restore)
1208 error = pm->restore(dev);
1209 } else {
1210 pci_pm_reenable_device(pci_dev);
1211 }
1212
1213 return error;
1214}
1215
1216#else /* !CONFIG_HIBERNATE_CALLBACKS */
1217
1218#define pci_pm_freeze NULL
1219#define pci_pm_freeze_late NULL
1220#define pci_pm_freeze_noirq NULL
1221#define pci_pm_thaw NULL
1222#define pci_pm_thaw_noirq NULL
1223#define pci_pm_poweroff NULL
1224#define pci_pm_poweroff_late NULL
1225#define pci_pm_poweroff_noirq NULL
1226#define pci_pm_restore NULL
1227#define pci_pm_restore_noirq NULL
1228
1229#endif /* !CONFIG_HIBERNATE_CALLBACKS */
1230
1231#ifdef CONFIG_PM
1232
1233static int pci_pm_runtime_suspend(struct device *dev)
1234{
1235 struct pci_dev *pci_dev = to_pci_dev(dev);
1236 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1237 pci_power_t prev = pci_dev->current_state;
1238 int error;
1239
1240 /*
1241 * If pci_dev->driver is not set (unbound), we leave the device in D0,
1242 * but it may go to D3cold when the bridge above it runtime suspends.
1243 * Save its config space in case that happens.
1244 */
1245 if (!pci_dev->driver) {
1246 pci_save_state(pci_dev);
1247 return 0;
1248 }
1249
1250 if (!pm || !pm->runtime_suspend)
1251 return -ENOSYS;
1252
1253 pci_dev->state_saved = false;
1254 error = pm->runtime_suspend(dev);
1255 if (error) {
1256 /*
1257 * -EBUSY and -EAGAIN is used to request the runtime PM core
1258 * to schedule a new suspend, so log the event only with debug
1259 * log level.
1260 */
1261 if (error == -EBUSY || error == -EAGAIN)
1262 dev_dbg(dev, "can't suspend now (%pf returned %d)\n",
1263 pm->runtime_suspend, error);
1264 else
1265 dev_err(dev, "can't suspend (%pf returned %d)\n",
1266 pm->runtime_suspend, error);
1267
1268 return error;
1269 }
1270
1271 pci_fixup_device(pci_fixup_suspend, pci_dev);
1272
1273 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1274 && pci_dev->current_state != PCI_UNKNOWN) {
1275 WARN_ONCE(pci_dev->current_state != prev,
1276 "PCI PM: State of device not saved by %pF\n",
1277 pm->runtime_suspend);
1278 return 0;
1279 }
1280
1281 if (!pci_dev->state_saved) {
1282 pci_save_state(pci_dev);
1283 pci_finish_runtime_suspend(pci_dev);
1284 }
1285
1286 return 0;
1287}
1288
1289static int pci_pm_runtime_resume(struct device *dev)
1290{
1291 int rc;
1292 struct pci_dev *pci_dev = to_pci_dev(dev);
1293 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1294
1295 /*
1296 * Restoring config space is necessary even if the device is not bound
1297 * to a driver because although we left it in D0, it may have gone to
1298 * D3cold when the bridge above it runtime suspended.
1299 */
1300 pci_restore_standard_config(pci_dev);
1301
1302 if (!pci_dev->driver)
1303 return 0;
1304
1305 if (!pm || !pm->runtime_resume)
1306 return -ENOSYS;
1307
1308 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1309 pci_enable_wake(pci_dev, PCI_D0, false);
1310 pci_fixup_device(pci_fixup_resume, pci_dev);
1311
1312 rc = pm->runtime_resume(dev);
1313
1314 pci_dev->runtime_d3cold = false;
1315
1316 return rc;
1317}
1318
1319static int pci_pm_runtime_idle(struct device *dev)
1320{
1321 struct pci_dev *pci_dev = to_pci_dev(dev);
1322 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1323 int ret = 0;
1324
1325 /*
1326 * If pci_dev->driver is not set (unbound), the device should
1327 * always remain in D0 regardless of the runtime PM status
1328 */
1329 if (!pci_dev->driver)
1330 return 0;
1331
1332 if (!pm)
1333 return -ENOSYS;
1334
1335 if (pm->runtime_idle)
1336 ret = pm->runtime_idle(dev);
1337
1338 return ret;
1339}
1340
1341static const struct dev_pm_ops pci_dev_pm_ops = {
1342 .prepare = pci_pm_prepare,
1343 .complete = pci_pm_complete,
1344 .suspend = pci_pm_suspend,
1345 .suspend_late = pci_pm_suspend_late,
1346 .resume = pci_pm_resume,
1347 .freeze = pci_pm_freeze,
1348 .freeze_late = pci_pm_freeze_late,
1349 .thaw = pci_pm_thaw,
1350 .poweroff = pci_pm_poweroff,
1351 .poweroff_late = pci_pm_poweroff_late,
1352 .restore = pci_pm_restore,
1353 .suspend_noirq = pci_pm_suspend_noirq,
1354 .resume_noirq = pci_pm_resume_noirq,
1355 .freeze_noirq = pci_pm_freeze_noirq,
1356 .thaw_noirq = pci_pm_thaw_noirq,
1357 .poweroff_noirq = pci_pm_poweroff_noirq,
1358 .restore_noirq = pci_pm_restore_noirq,
1359 .runtime_suspend = pci_pm_runtime_suspend,
1360 .runtime_resume = pci_pm_runtime_resume,
1361 .runtime_idle = pci_pm_runtime_idle,
1362};
1363
1364#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1365
1366#else /* !CONFIG_PM */
1367
1368#define pci_pm_runtime_suspend NULL
1369#define pci_pm_runtime_resume NULL
1370#define pci_pm_runtime_idle NULL
1371
1372#define PCI_PM_OPS_PTR NULL
1373
1374#endif /* !CONFIG_PM */
1375
1376/**
1377 * __pci_register_driver - register a new pci driver
1378 * @drv: the driver structure to register
1379 * @owner: owner module of drv
1380 * @mod_name: module name string
1381 *
1382 * Adds the driver structure to the list of registered drivers.
1383 * Returns a negative value on error, otherwise 0.
1384 * If no error occurred, the driver remains registered even if
1385 * no device was claimed during registration.
1386 */
1387int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1388 const char *mod_name)
1389{
1390 /* initialize common driver fields */
1391 drv->driver.name = drv->name;
1392 drv->driver.bus = &pci_bus_type;
1393 drv->driver.owner = owner;
1394 drv->driver.mod_name = mod_name;
1395 drv->driver.groups = drv->groups;
1396
1397 spin_lock_init(&drv->dynids.lock);
1398 INIT_LIST_HEAD(&drv->dynids.list);
1399
1400 /* register with core */
1401 return driver_register(&drv->driver);
1402}
1403EXPORT_SYMBOL(__pci_register_driver);
1404
1405/**
1406 * pci_unregister_driver - unregister a pci driver
1407 * @drv: the driver structure to unregister
1408 *
1409 * Deletes the driver structure from the list of registered PCI drivers,
1410 * gives it a chance to clean up by calling its remove() function for
1411 * each device it was responsible for, and marks those devices as
1412 * driverless.
1413 */
1414
1415void pci_unregister_driver(struct pci_driver *drv)
1416{
1417 driver_unregister(&drv->driver);
1418 pci_free_dynids(drv);
1419}
1420EXPORT_SYMBOL(pci_unregister_driver);
1421
1422static struct pci_driver pci_compat_driver = {
1423 .name = "compat"
1424};
1425
1426/**
1427 * pci_dev_driver - get the pci_driver of a device
1428 * @dev: the device to query
1429 *
1430 * Returns the appropriate pci_driver structure or %NULL if there is no
1431 * registered driver for the device.
1432 */
1433struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1434{
1435 if (dev->driver)
1436 return dev->driver;
1437 else {
1438 int i;
1439 for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1440 if (dev->resource[i].flags & IORESOURCE_BUSY)
1441 return &pci_compat_driver;
1442 }
1443 return NULL;
1444}
1445EXPORT_SYMBOL(pci_dev_driver);
1446
1447/**
1448 * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1449 * @dev: the PCI device structure to match against
1450 * @drv: the device driver to search for matching PCI device id structures
1451 *
1452 * Used by a driver to check whether a PCI device present in the
1453 * system is in its list of supported devices. Returns the matching
1454 * pci_device_id structure or %NULL if there is no match.
1455 */
1456static int pci_bus_match(struct device *dev, struct device_driver *drv)
1457{
1458 struct pci_dev *pci_dev = to_pci_dev(dev);
1459 struct pci_driver *pci_drv;
1460 const struct pci_device_id *found_id;
1461
1462 if (!pci_dev->match_driver)
1463 return 0;
1464
1465 pci_drv = to_pci_driver(drv);
1466 found_id = pci_match_device(pci_drv, pci_dev);
1467 if (found_id)
1468 return 1;
1469
1470 return 0;
1471}
1472
1473/**
1474 * pci_dev_get - increments the reference count of the pci device structure
1475 * @dev: the device being referenced
1476 *
1477 * Each live reference to a device should be refcounted.
1478 *
1479 * Drivers for PCI devices should normally record such references in
1480 * their probe() methods, when they bind to a device, and release
1481 * them by calling pci_dev_put(), in their disconnect() methods.
1482 *
1483 * A pointer to the device with the incremented reference counter is returned.
1484 */
1485struct pci_dev *pci_dev_get(struct pci_dev *dev)
1486{
1487 if (dev)
1488 get_device(&dev->dev);
1489 return dev;
1490}
1491EXPORT_SYMBOL(pci_dev_get);
1492
1493/**
1494 * pci_dev_put - release a use of the pci device structure
1495 * @dev: device that's been disconnected
1496 *
1497 * Must be called when a user of a device is finished with it. When the last
1498 * user of the device calls this function, the memory of the device is freed.
1499 */
1500void pci_dev_put(struct pci_dev *dev)
1501{
1502 if (dev)
1503 put_device(&dev->dev);
1504}
1505EXPORT_SYMBOL(pci_dev_put);
1506
1507static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1508{
1509 struct pci_dev *pdev;
1510
1511 if (!dev)
1512 return -ENODEV;
1513
1514 pdev = to_pci_dev(dev);
1515
1516 if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1517 return -ENOMEM;
1518
1519 if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1520 return -ENOMEM;
1521
1522 if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1523 pdev->subsystem_device))
1524 return -ENOMEM;
1525
1526 if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1527 return -ENOMEM;
1528
1529 if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1530 pdev->vendor, pdev->device,
1531 pdev->subsystem_vendor, pdev->subsystem_device,
1532 (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1533 (u8)(pdev->class)))
1534 return -ENOMEM;
1535
1536 return 0;
1537}
1538
1539#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH)
1540/**
1541 * pci_uevent_ers - emit a uevent during recovery path of PCI device
1542 * @pdev: PCI device undergoing error recovery
1543 * @err_type: type of error event
1544 */
1545void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1546{
1547 int idx = 0;
1548 char *envp[3];
1549
1550 switch (err_type) {
1551 case PCI_ERS_RESULT_NONE:
1552 case PCI_ERS_RESULT_CAN_RECOVER:
1553 envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1554 envp[idx++] = "DEVICE_ONLINE=0";
1555 break;
1556 case PCI_ERS_RESULT_RECOVERED:
1557 envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1558 envp[idx++] = "DEVICE_ONLINE=1";
1559 break;
1560 case PCI_ERS_RESULT_DISCONNECT:
1561 envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1562 envp[idx++] = "DEVICE_ONLINE=0";
1563 break;
1564 default:
1565 break;
1566 }
1567
1568 if (idx > 0) {
1569 envp[idx++] = NULL;
1570 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1571 }
1572}
1573#endif
1574
1575static int pci_bus_num_vf(struct device *dev)
1576{
1577 return pci_num_vf(to_pci_dev(dev));
1578}
1579
1580struct bus_type pci_bus_type = {
1581 .name = "pci",
1582 .match = pci_bus_match,
1583 .uevent = pci_uevent,
1584 .probe = pci_device_probe,
1585 .remove = pci_device_remove,
1586 .shutdown = pci_device_shutdown,
1587 .dev_groups = pci_dev_groups,
1588 .bus_groups = pci_bus_groups,
1589 .drv_groups = pci_drv_groups,
1590 .pm = PCI_PM_OPS_PTR,
1591 .num_vf = pci_bus_num_vf,
1592 .force_dma = true,
1593};
1594EXPORT_SYMBOL(pci_bus_type);
1595
1596#ifdef CONFIG_PCIEPORTBUS
1597static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
1598{
1599 struct pcie_device *pciedev;
1600 struct pcie_port_service_driver *driver;
1601
1602 if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1603 return 0;
1604
1605 pciedev = to_pcie_device(dev);
1606 driver = to_service_driver(drv);
1607
1608 if (driver->service != pciedev->service)
1609 return 0;
1610
1611 if (driver->port_type != PCIE_ANY_PORT &&
1612 driver->port_type != pci_pcie_type(pciedev->port))
1613 return 0;
1614
1615 return 1;
1616}
1617
1618struct bus_type pcie_port_bus_type = {
1619 .name = "pci_express",
1620 .match = pcie_port_bus_match,
1621};
1622EXPORT_SYMBOL_GPL(pcie_port_bus_type);
1623#endif
1624
1625static int __init pci_driver_init(void)
1626{
1627 int ret;
1628
1629 ret = bus_register(&pci_bus_type);
1630 if (ret)
1631 return ret;
1632
1633#ifdef CONFIG_PCIEPORTBUS
1634 ret = bus_register(&pcie_port_bus_type);
1635 if (ret)
1636 return ret;
1637#endif
1638
1639 return 0;
1640}
1641postcore_initcall(pci_driver_init);