Loading...
1/*
2 * drivers/base/power/main.c - Where the driver meets power management.
3 *
4 * Copyright (c) 2003 Patrick Mochel
5 * Copyright (c) 2003 Open Source Development Lab
6 *
7 * This file is released under the GPLv2
8 *
9 *
10 * The driver model core calls device_pm_add() when a device is registered.
11 * This will initialize the embedded device_pm_info object in the device
12 * and add it to the list of power-controlled devices. sysfs entries for
13 * controlling device power management will also be added.
14 *
15 * A separate list is used for keeping track of power info, because the power
16 * domain dependencies may differ from the ancestral dependencies that the
17 * subsystem list maintains.
18 */
19
20#include <linux/device.h>
21#include <linux/kallsyms.h>
22#include <linux/export.h>
23#include <linux/mutex.h>
24#include <linux/pm.h>
25#include <linux/pm_runtime.h>
26#include <linux/resume-trace.h>
27#include <linux/interrupt.h>
28#include <linux/sched.h>
29#include <linux/async.h>
30#include <linux/suspend.h>
31#include <trace/events/power.h>
32#include <linux/cpufreq.h>
33#include <linux/cpuidle.h>
34#include <linux/timer.h>
35
36#include "../base.h"
37#include "power.h"
38
39typedef int (*pm_callback_t)(struct device *);
40
41/*
42 * The entries in the dpm_list list are in a depth first order, simply
43 * because children are guaranteed to be discovered after parents, and
44 * are inserted at the back of the list on discovery.
45 *
46 * Since device_pm_add() may be called with a device lock held,
47 * we must never try to acquire a device lock while holding
48 * dpm_list_mutex.
49 */
50
51LIST_HEAD(dpm_list);
52static LIST_HEAD(dpm_prepared_list);
53static LIST_HEAD(dpm_suspended_list);
54static LIST_HEAD(dpm_late_early_list);
55static LIST_HEAD(dpm_noirq_list);
56
57struct suspend_stats suspend_stats;
58static DEFINE_MUTEX(dpm_list_mtx);
59static pm_message_t pm_transition;
60
61static int async_error;
62
63static char *pm_verb(int event)
64{
65 switch (event) {
66 case PM_EVENT_SUSPEND:
67 return "suspend";
68 case PM_EVENT_RESUME:
69 return "resume";
70 case PM_EVENT_FREEZE:
71 return "freeze";
72 case PM_EVENT_QUIESCE:
73 return "quiesce";
74 case PM_EVENT_HIBERNATE:
75 return "hibernate";
76 case PM_EVENT_THAW:
77 return "thaw";
78 case PM_EVENT_RESTORE:
79 return "restore";
80 case PM_EVENT_RECOVER:
81 return "recover";
82 default:
83 return "(unknown PM event)";
84 }
85}
86
87/**
88 * device_pm_sleep_init - Initialize system suspend-related device fields.
89 * @dev: Device object being initialized.
90 */
91void device_pm_sleep_init(struct device *dev)
92{
93 dev->power.is_prepared = false;
94 dev->power.is_suspended = false;
95 dev->power.is_noirq_suspended = false;
96 dev->power.is_late_suspended = false;
97 init_completion(&dev->power.completion);
98 complete_all(&dev->power.completion);
99 dev->power.wakeup = NULL;
100 INIT_LIST_HEAD(&dev->power.entry);
101}
102
103/**
104 * device_pm_lock - Lock the list of active devices used by the PM core.
105 */
106void device_pm_lock(void)
107{
108 mutex_lock(&dpm_list_mtx);
109}
110
111/**
112 * device_pm_unlock - Unlock the list of active devices used by the PM core.
113 */
114void device_pm_unlock(void)
115{
116 mutex_unlock(&dpm_list_mtx);
117}
118
119/**
120 * device_pm_add - Add a device to the PM core's list of active devices.
121 * @dev: Device to add to the list.
122 */
123void device_pm_add(struct device *dev)
124{
125 pr_debug("PM: Adding info for %s:%s\n",
126 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
127 mutex_lock(&dpm_list_mtx);
128 if (dev->parent && dev->parent->power.is_prepared)
129 dev_warn(dev, "parent %s should not be sleeping\n",
130 dev_name(dev->parent));
131 list_add_tail(&dev->power.entry, &dpm_list);
132 mutex_unlock(&dpm_list_mtx);
133}
134
135/**
136 * device_pm_remove - Remove a device from the PM core's list of active devices.
137 * @dev: Device to be removed from the list.
138 */
139void device_pm_remove(struct device *dev)
140{
141 pr_debug("PM: Removing info for %s:%s\n",
142 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
143 complete_all(&dev->power.completion);
144 mutex_lock(&dpm_list_mtx);
145 list_del_init(&dev->power.entry);
146 mutex_unlock(&dpm_list_mtx);
147 device_wakeup_disable(dev);
148 pm_runtime_remove(dev);
149}
150
151/**
152 * device_pm_move_before - Move device in the PM core's list of active devices.
153 * @deva: Device to move in dpm_list.
154 * @devb: Device @deva should come before.
155 */
156void device_pm_move_before(struct device *deva, struct device *devb)
157{
158 pr_debug("PM: Moving %s:%s before %s:%s\n",
159 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
160 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
161 /* Delete deva from dpm_list and reinsert before devb. */
162 list_move_tail(&deva->power.entry, &devb->power.entry);
163}
164
165/**
166 * device_pm_move_after - Move device in the PM core's list of active devices.
167 * @deva: Device to move in dpm_list.
168 * @devb: Device @deva should come after.
169 */
170void device_pm_move_after(struct device *deva, struct device *devb)
171{
172 pr_debug("PM: Moving %s:%s after %s:%s\n",
173 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
174 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
175 /* Delete deva from dpm_list and reinsert after devb. */
176 list_move(&deva->power.entry, &devb->power.entry);
177}
178
179/**
180 * device_pm_move_last - Move device to end of the PM core's list of devices.
181 * @dev: Device to move in dpm_list.
182 */
183void device_pm_move_last(struct device *dev)
184{
185 pr_debug("PM: Moving %s:%s to end of list\n",
186 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
187 list_move_tail(&dev->power.entry, &dpm_list);
188}
189
190static ktime_t initcall_debug_start(struct device *dev)
191{
192 ktime_t calltime = ktime_set(0, 0);
193
194 if (pm_print_times_enabled) {
195 pr_info("calling %s+ @ %i, parent: %s\n",
196 dev_name(dev), task_pid_nr(current),
197 dev->parent ? dev_name(dev->parent) : "none");
198 calltime = ktime_get();
199 }
200
201 return calltime;
202}
203
204static void initcall_debug_report(struct device *dev, ktime_t calltime,
205 int error, pm_message_t state, char *info)
206{
207 ktime_t rettime;
208 s64 nsecs;
209
210 rettime = ktime_get();
211 nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
212
213 if (pm_print_times_enabled) {
214 pr_info("call %s+ returned %d after %Ld usecs\n", dev_name(dev),
215 error, (unsigned long long)nsecs >> 10);
216 }
217
218 trace_device_pm_report_time(dev, info, nsecs, pm_verb(state.event),
219 error);
220}
221
222/**
223 * dpm_wait - Wait for a PM operation to complete.
224 * @dev: Device to wait for.
225 * @async: If unset, wait only if the device's power.async_suspend flag is set.
226 */
227static void dpm_wait(struct device *dev, bool async)
228{
229 if (!dev)
230 return;
231
232 if (async || (pm_async_enabled && dev->power.async_suspend))
233 wait_for_completion(&dev->power.completion);
234}
235
236static int dpm_wait_fn(struct device *dev, void *async_ptr)
237{
238 dpm_wait(dev, *((bool *)async_ptr));
239 return 0;
240}
241
242static void dpm_wait_for_children(struct device *dev, bool async)
243{
244 device_for_each_child(dev, &async, dpm_wait_fn);
245}
246
247/**
248 * pm_op - Return the PM operation appropriate for given PM event.
249 * @ops: PM operations to choose from.
250 * @state: PM transition of the system being carried out.
251 */
252static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
253{
254 switch (state.event) {
255#ifdef CONFIG_SUSPEND
256 case PM_EVENT_SUSPEND:
257 return ops->suspend;
258 case PM_EVENT_RESUME:
259 return ops->resume;
260#endif /* CONFIG_SUSPEND */
261#ifdef CONFIG_HIBERNATE_CALLBACKS
262 case PM_EVENT_FREEZE:
263 case PM_EVENT_QUIESCE:
264 return ops->freeze;
265 case PM_EVENT_HIBERNATE:
266 return ops->poweroff;
267 case PM_EVENT_THAW:
268 case PM_EVENT_RECOVER:
269 return ops->thaw;
270 break;
271 case PM_EVENT_RESTORE:
272 return ops->restore;
273#endif /* CONFIG_HIBERNATE_CALLBACKS */
274 }
275
276 return NULL;
277}
278
279/**
280 * pm_late_early_op - Return the PM operation appropriate for given PM event.
281 * @ops: PM operations to choose from.
282 * @state: PM transition of the system being carried out.
283 *
284 * Runtime PM is disabled for @dev while this function is being executed.
285 */
286static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
287 pm_message_t state)
288{
289 switch (state.event) {
290#ifdef CONFIG_SUSPEND
291 case PM_EVENT_SUSPEND:
292 return ops->suspend_late;
293 case PM_EVENT_RESUME:
294 return ops->resume_early;
295#endif /* CONFIG_SUSPEND */
296#ifdef CONFIG_HIBERNATE_CALLBACKS
297 case PM_EVENT_FREEZE:
298 case PM_EVENT_QUIESCE:
299 return ops->freeze_late;
300 case PM_EVENT_HIBERNATE:
301 return ops->poweroff_late;
302 case PM_EVENT_THAW:
303 case PM_EVENT_RECOVER:
304 return ops->thaw_early;
305 case PM_EVENT_RESTORE:
306 return ops->restore_early;
307#endif /* CONFIG_HIBERNATE_CALLBACKS */
308 }
309
310 return NULL;
311}
312
313/**
314 * pm_noirq_op - Return the PM operation appropriate for given PM event.
315 * @ops: PM operations to choose from.
316 * @state: PM transition of the system being carried out.
317 *
318 * The driver of @dev will not receive interrupts while this function is being
319 * executed.
320 */
321static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
322{
323 switch (state.event) {
324#ifdef CONFIG_SUSPEND
325 case PM_EVENT_SUSPEND:
326 return ops->suspend_noirq;
327 case PM_EVENT_RESUME:
328 return ops->resume_noirq;
329#endif /* CONFIG_SUSPEND */
330#ifdef CONFIG_HIBERNATE_CALLBACKS
331 case PM_EVENT_FREEZE:
332 case PM_EVENT_QUIESCE:
333 return ops->freeze_noirq;
334 case PM_EVENT_HIBERNATE:
335 return ops->poweroff_noirq;
336 case PM_EVENT_THAW:
337 case PM_EVENT_RECOVER:
338 return ops->thaw_noirq;
339 case PM_EVENT_RESTORE:
340 return ops->restore_noirq;
341#endif /* CONFIG_HIBERNATE_CALLBACKS */
342 }
343
344 return NULL;
345}
346
347static void pm_dev_dbg(struct device *dev, pm_message_t state, char *info)
348{
349 dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event),
350 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
351 ", may wakeup" : "");
352}
353
354static void pm_dev_err(struct device *dev, pm_message_t state, char *info,
355 int error)
356{
357 printk(KERN_ERR "PM: Device %s failed to %s%s: error %d\n",
358 dev_name(dev), pm_verb(state.event), info, error);
359}
360
361static void dpm_show_time(ktime_t starttime, pm_message_t state, char *info)
362{
363 ktime_t calltime;
364 u64 usecs64;
365 int usecs;
366
367 calltime = ktime_get();
368 usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
369 do_div(usecs64, NSEC_PER_USEC);
370 usecs = usecs64;
371 if (usecs == 0)
372 usecs = 1;
373 pr_info("PM: %s%s%s of devices complete after %ld.%03ld msecs\n",
374 info ?: "", info ? " " : "", pm_verb(state.event),
375 usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
376}
377
378static int dpm_run_callback(pm_callback_t cb, struct device *dev,
379 pm_message_t state, char *info)
380{
381 ktime_t calltime;
382 int error;
383
384 if (!cb)
385 return 0;
386
387 calltime = initcall_debug_start(dev);
388
389 pm_dev_dbg(dev, state, info);
390 error = cb(dev);
391 suspend_report_result(cb, error);
392
393 initcall_debug_report(dev, calltime, error, state, info);
394
395 return error;
396}
397
398#ifdef CONFIG_DPM_WATCHDOG
399struct dpm_watchdog {
400 struct device *dev;
401 struct task_struct *tsk;
402 struct timer_list timer;
403};
404
405#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
406 struct dpm_watchdog wd
407
408/**
409 * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
410 * @data: Watchdog object address.
411 *
412 * Called when a driver has timed out suspending or resuming.
413 * There's not much we can do here to recover so panic() to
414 * capture a crash-dump in pstore.
415 */
416static void dpm_watchdog_handler(unsigned long data)
417{
418 struct dpm_watchdog *wd = (void *)data;
419
420 dev_emerg(wd->dev, "**** DPM device timeout ****\n");
421 show_stack(wd->tsk, NULL);
422 panic("%s %s: unrecoverable failure\n",
423 dev_driver_string(wd->dev), dev_name(wd->dev));
424}
425
426/**
427 * dpm_watchdog_set - Enable pm watchdog for given device.
428 * @wd: Watchdog. Must be allocated on the stack.
429 * @dev: Device to handle.
430 */
431static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
432{
433 struct timer_list *timer = &wd->timer;
434
435 wd->dev = dev;
436 wd->tsk = current;
437
438 init_timer_on_stack(timer);
439 /* use same timeout value for both suspend and resume */
440 timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
441 timer->function = dpm_watchdog_handler;
442 timer->data = (unsigned long)wd;
443 add_timer(timer);
444}
445
446/**
447 * dpm_watchdog_clear - Disable suspend/resume watchdog.
448 * @wd: Watchdog to disable.
449 */
450static void dpm_watchdog_clear(struct dpm_watchdog *wd)
451{
452 struct timer_list *timer = &wd->timer;
453
454 del_timer_sync(timer);
455 destroy_timer_on_stack(timer);
456}
457#else
458#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
459#define dpm_watchdog_set(x, y)
460#define dpm_watchdog_clear(x)
461#endif
462
463/*------------------------- Resume routines -------------------------*/
464
465/**
466 * device_resume_noirq - Execute an "early resume" callback for given device.
467 * @dev: Device to handle.
468 * @state: PM transition of the system being carried out.
469 *
470 * The driver of @dev will not receive interrupts while this function is being
471 * executed.
472 */
473static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
474{
475 pm_callback_t callback = NULL;
476 char *info = NULL;
477 int error = 0;
478
479 TRACE_DEVICE(dev);
480 TRACE_RESUME(0);
481
482 if (dev->power.syscore)
483 goto Out;
484
485 if (!dev->power.is_noirq_suspended)
486 goto Out;
487
488 dpm_wait(dev->parent, async);
489
490 if (dev->pm_domain) {
491 info = "noirq power domain ";
492 callback = pm_noirq_op(&dev->pm_domain->ops, state);
493 } else if (dev->type && dev->type->pm) {
494 info = "noirq type ";
495 callback = pm_noirq_op(dev->type->pm, state);
496 } else if (dev->class && dev->class->pm) {
497 info = "noirq class ";
498 callback = pm_noirq_op(dev->class->pm, state);
499 } else if (dev->bus && dev->bus->pm) {
500 info = "noirq bus ";
501 callback = pm_noirq_op(dev->bus->pm, state);
502 }
503
504 if (!callback && dev->driver && dev->driver->pm) {
505 info = "noirq driver ";
506 callback = pm_noirq_op(dev->driver->pm, state);
507 }
508
509 error = dpm_run_callback(callback, dev, state, info);
510 dev->power.is_noirq_suspended = false;
511
512 Out:
513 complete_all(&dev->power.completion);
514 TRACE_RESUME(error);
515 return error;
516}
517
518static bool is_async(struct device *dev)
519{
520 return dev->power.async_suspend && pm_async_enabled
521 && !pm_trace_is_enabled();
522}
523
524static void async_resume_noirq(void *data, async_cookie_t cookie)
525{
526 struct device *dev = (struct device *)data;
527 int error;
528
529 error = device_resume_noirq(dev, pm_transition, true);
530 if (error)
531 pm_dev_err(dev, pm_transition, " async", error);
532
533 put_device(dev);
534}
535
536/**
537 * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
538 * @state: PM transition of the system being carried out.
539 *
540 * Call the "noirq" resume handlers for all devices in dpm_noirq_list and
541 * enable device drivers to receive interrupts.
542 */
543static void dpm_resume_noirq(pm_message_t state)
544{
545 struct device *dev;
546 ktime_t starttime = ktime_get();
547
548 mutex_lock(&dpm_list_mtx);
549 pm_transition = state;
550
551 /*
552 * Advanced the async threads upfront,
553 * in case the starting of async threads is
554 * delayed by non-async resuming devices.
555 */
556 list_for_each_entry(dev, &dpm_noirq_list, power.entry) {
557 reinit_completion(&dev->power.completion);
558 if (is_async(dev)) {
559 get_device(dev);
560 async_schedule(async_resume_noirq, dev);
561 }
562 }
563
564 while (!list_empty(&dpm_noirq_list)) {
565 dev = to_device(dpm_noirq_list.next);
566 get_device(dev);
567 list_move_tail(&dev->power.entry, &dpm_late_early_list);
568 mutex_unlock(&dpm_list_mtx);
569
570 if (!is_async(dev)) {
571 int error;
572
573 error = device_resume_noirq(dev, state, false);
574 if (error) {
575 suspend_stats.failed_resume_noirq++;
576 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
577 dpm_save_failed_dev(dev_name(dev));
578 pm_dev_err(dev, state, " noirq", error);
579 }
580 }
581
582 mutex_lock(&dpm_list_mtx);
583 put_device(dev);
584 }
585 mutex_unlock(&dpm_list_mtx);
586 async_synchronize_full();
587 dpm_show_time(starttime, state, "noirq");
588 resume_device_irqs();
589 cpuidle_resume();
590}
591
592/**
593 * device_resume_early - Execute an "early resume" callback for given device.
594 * @dev: Device to handle.
595 * @state: PM transition of the system being carried out.
596 *
597 * Runtime PM is disabled for @dev while this function is being executed.
598 */
599static int device_resume_early(struct device *dev, pm_message_t state, bool async)
600{
601 pm_callback_t callback = NULL;
602 char *info = NULL;
603 int error = 0;
604
605 TRACE_DEVICE(dev);
606 TRACE_RESUME(0);
607
608 if (dev->power.syscore)
609 goto Out;
610
611 if (!dev->power.is_late_suspended)
612 goto Out;
613
614 dpm_wait(dev->parent, async);
615
616 if (dev->pm_domain) {
617 info = "early power domain ";
618 callback = pm_late_early_op(&dev->pm_domain->ops, state);
619 } else if (dev->type && dev->type->pm) {
620 info = "early type ";
621 callback = pm_late_early_op(dev->type->pm, state);
622 } else if (dev->class && dev->class->pm) {
623 info = "early class ";
624 callback = pm_late_early_op(dev->class->pm, state);
625 } else if (dev->bus && dev->bus->pm) {
626 info = "early bus ";
627 callback = pm_late_early_op(dev->bus->pm, state);
628 }
629
630 if (!callback && dev->driver && dev->driver->pm) {
631 info = "early driver ";
632 callback = pm_late_early_op(dev->driver->pm, state);
633 }
634
635 error = dpm_run_callback(callback, dev, state, info);
636 dev->power.is_late_suspended = false;
637
638 Out:
639 TRACE_RESUME(error);
640
641 pm_runtime_enable(dev);
642 complete_all(&dev->power.completion);
643 return error;
644}
645
646static void async_resume_early(void *data, async_cookie_t cookie)
647{
648 struct device *dev = (struct device *)data;
649 int error;
650
651 error = device_resume_early(dev, pm_transition, true);
652 if (error)
653 pm_dev_err(dev, pm_transition, " async", error);
654
655 put_device(dev);
656}
657
658/**
659 * dpm_resume_early - Execute "early resume" callbacks for all devices.
660 * @state: PM transition of the system being carried out.
661 */
662static void dpm_resume_early(pm_message_t state)
663{
664 struct device *dev;
665 ktime_t starttime = ktime_get();
666
667 mutex_lock(&dpm_list_mtx);
668 pm_transition = state;
669
670 /*
671 * Advanced the async threads upfront,
672 * in case the starting of async threads is
673 * delayed by non-async resuming devices.
674 */
675 list_for_each_entry(dev, &dpm_late_early_list, power.entry) {
676 reinit_completion(&dev->power.completion);
677 if (is_async(dev)) {
678 get_device(dev);
679 async_schedule(async_resume_early, dev);
680 }
681 }
682
683 while (!list_empty(&dpm_late_early_list)) {
684 dev = to_device(dpm_late_early_list.next);
685 get_device(dev);
686 list_move_tail(&dev->power.entry, &dpm_suspended_list);
687 mutex_unlock(&dpm_list_mtx);
688
689 if (!is_async(dev)) {
690 int error;
691
692 error = device_resume_early(dev, state, false);
693 if (error) {
694 suspend_stats.failed_resume_early++;
695 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
696 dpm_save_failed_dev(dev_name(dev));
697 pm_dev_err(dev, state, " early", error);
698 }
699 }
700 mutex_lock(&dpm_list_mtx);
701 put_device(dev);
702 }
703 mutex_unlock(&dpm_list_mtx);
704 async_synchronize_full();
705 dpm_show_time(starttime, state, "early");
706}
707
708/**
709 * dpm_resume_start - Execute "noirq" and "early" device callbacks.
710 * @state: PM transition of the system being carried out.
711 */
712void dpm_resume_start(pm_message_t state)
713{
714 dpm_resume_noirq(state);
715 dpm_resume_early(state);
716}
717EXPORT_SYMBOL_GPL(dpm_resume_start);
718
719/**
720 * device_resume - Execute "resume" callbacks for given device.
721 * @dev: Device to handle.
722 * @state: PM transition of the system being carried out.
723 * @async: If true, the device is being resumed asynchronously.
724 */
725static int device_resume(struct device *dev, pm_message_t state, bool async)
726{
727 pm_callback_t callback = NULL;
728 char *info = NULL;
729 int error = 0;
730 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
731
732 TRACE_DEVICE(dev);
733 TRACE_RESUME(0);
734
735 if (dev->power.syscore)
736 goto Complete;
737
738 dpm_wait(dev->parent, async);
739 dpm_watchdog_set(&wd, dev);
740 device_lock(dev);
741
742 /*
743 * This is a fib. But we'll allow new children to be added below
744 * a resumed device, even if the device hasn't been completed yet.
745 */
746 dev->power.is_prepared = false;
747
748 if (!dev->power.is_suspended)
749 goto Unlock;
750
751 if (dev->pm_domain) {
752 info = "power domain ";
753 callback = pm_op(&dev->pm_domain->ops, state);
754 goto Driver;
755 }
756
757 if (dev->type && dev->type->pm) {
758 info = "type ";
759 callback = pm_op(dev->type->pm, state);
760 goto Driver;
761 }
762
763 if (dev->class) {
764 if (dev->class->pm) {
765 info = "class ";
766 callback = pm_op(dev->class->pm, state);
767 goto Driver;
768 } else if (dev->class->resume) {
769 info = "legacy class ";
770 callback = dev->class->resume;
771 goto End;
772 }
773 }
774
775 if (dev->bus) {
776 if (dev->bus->pm) {
777 info = "bus ";
778 callback = pm_op(dev->bus->pm, state);
779 } else if (dev->bus->resume) {
780 info = "legacy bus ";
781 callback = dev->bus->resume;
782 goto End;
783 }
784 }
785
786 Driver:
787 if (!callback && dev->driver && dev->driver->pm) {
788 info = "driver ";
789 callback = pm_op(dev->driver->pm, state);
790 }
791
792 End:
793 error = dpm_run_callback(callback, dev, state, info);
794 dev->power.is_suspended = false;
795
796 Unlock:
797 device_unlock(dev);
798 dpm_watchdog_clear(&wd);
799
800 Complete:
801 complete_all(&dev->power.completion);
802
803 TRACE_RESUME(error);
804
805 return error;
806}
807
808static void async_resume(void *data, async_cookie_t cookie)
809{
810 struct device *dev = (struct device *)data;
811 int error;
812
813 error = device_resume(dev, pm_transition, true);
814 if (error)
815 pm_dev_err(dev, pm_transition, " async", error);
816 put_device(dev);
817}
818
819/**
820 * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
821 * @state: PM transition of the system being carried out.
822 *
823 * Execute the appropriate "resume" callback for all devices whose status
824 * indicates that they are suspended.
825 */
826void dpm_resume(pm_message_t state)
827{
828 struct device *dev;
829 ktime_t starttime = ktime_get();
830
831 might_sleep();
832
833 mutex_lock(&dpm_list_mtx);
834 pm_transition = state;
835 async_error = 0;
836
837 list_for_each_entry(dev, &dpm_suspended_list, power.entry) {
838 reinit_completion(&dev->power.completion);
839 if (is_async(dev)) {
840 get_device(dev);
841 async_schedule(async_resume, dev);
842 }
843 }
844
845 while (!list_empty(&dpm_suspended_list)) {
846 dev = to_device(dpm_suspended_list.next);
847 get_device(dev);
848 if (!is_async(dev)) {
849 int error;
850
851 mutex_unlock(&dpm_list_mtx);
852
853 error = device_resume(dev, state, false);
854 if (error) {
855 suspend_stats.failed_resume++;
856 dpm_save_failed_step(SUSPEND_RESUME);
857 dpm_save_failed_dev(dev_name(dev));
858 pm_dev_err(dev, state, "", error);
859 }
860
861 mutex_lock(&dpm_list_mtx);
862 }
863 if (!list_empty(&dev->power.entry))
864 list_move_tail(&dev->power.entry, &dpm_prepared_list);
865 put_device(dev);
866 }
867 mutex_unlock(&dpm_list_mtx);
868 async_synchronize_full();
869 dpm_show_time(starttime, state, NULL);
870
871 cpufreq_resume();
872}
873
874/**
875 * device_complete - Complete a PM transition for given device.
876 * @dev: Device to handle.
877 * @state: PM transition of the system being carried out.
878 */
879static void device_complete(struct device *dev, pm_message_t state)
880{
881 void (*callback)(struct device *) = NULL;
882 char *info = NULL;
883
884 if (dev->power.syscore)
885 return;
886
887 device_lock(dev);
888
889 if (dev->pm_domain) {
890 info = "completing power domain ";
891 callback = dev->pm_domain->ops.complete;
892 } else if (dev->type && dev->type->pm) {
893 info = "completing type ";
894 callback = dev->type->pm->complete;
895 } else if (dev->class && dev->class->pm) {
896 info = "completing class ";
897 callback = dev->class->pm->complete;
898 } else if (dev->bus && dev->bus->pm) {
899 info = "completing bus ";
900 callback = dev->bus->pm->complete;
901 }
902
903 if (!callback && dev->driver && dev->driver->pm) {
904 info = "completing driver ";
905 callback = dev->driver->pm->complete;
906 }
907
908 if (callback) {
909 pm_dev_dbg(dev, state, info);
910 callback(dev);
911 }
912
913 device_unlock(dev);
914
915 pm_runtime_put(dev);
916}
917
918/**
919 * dpm_complete - Complete a PM transition for all non-sysdev devices.
920 * @state: PM transition of the system being carried out.
921 *
922 * Execute the ->complete() callbacks for all devices whose PM status is not
923 * DPM_ON (this allows new devices to be registered).
924 */
925void dpm_complete(pm_message_t state)
926{
927 struct list_head list;
928
929 might_sleep();
930
931 INIT_LIST_HEAD(&list);
932 mutex_lock(&dpm_list_mtx);
933 while (!list_empty(&dpm_prepared_list)) {
934 struct device *dev = to_device(dpm_prepared_list.prev);
935
936 get_device(dev);
937 dev->power.is_prepared = false;
938 list_move(&dev->power.entry, &list);
939 mutex_unlock(&dpm_list_mtx);
940
941 device_complete(dev, state);
942
943 mutex_lock(&dpm_list_mtx);
944 put_device(dev);
945 }
946 list_splice(&list, &dpm_list);
947 mutex_unlock(&dpm_list_mtx);
948}
949
950/**
951 * dpm_resume_end - Execute "resume" callbacks and complete system transition.
952 * @state: PM transition of the system being carried out.
953 *
954 * Execute "resume" callbacks for all devices and complete the PM transition of
955 * the system.
956 */
957void dpm_resume_end(pm_message_t state)
958{
959 dpm_resume(state);
960 dpm_complete(state);
961}
962EXPORT_SYMBOL_GPL(dpm_resume_end);
963
964
965/*------------------------- Suspend routines -------------------------*/
966
967/**
968 * resume_event - Return a "resume" message for given "suspend" sleep state.
969 * @sleep_state: PM message representing a sleep state.
970 *
971 * Return a PM message representing the resume event corresponding to given
972 * sleep state.
973 */
974static pm_message_t resume_event(pm_message_t sleep_state)
975{
976 switch (sleep_state.event) {
977 case PM_EVENT_SUSPEND:
978 return PMSG_RESUME;
979 case PM_EVENT_FREEZE:
980 case PM_EVENT_QUIESCE:
981 return PMSG_RECOVER;
982 case PM_EVENT_HIBERNATE:
983 return PMSG_RESTORE;
984 }
985 return PMSG_ON;
986}
987
988/**
989 * device_suspend_noirq - Execute a "late suspend" callback for given device.
990 * @dev: Device to handle.
991 * @state: PM transition of the system being carried out.
992 *
993 * The driver of @dev will not receive interrupts while this function is being
994 * executed.
995 */
996static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
997{
998 pm_callback_t callback = NULL;
999 char *info = NULL;
1000 int error = 0;
1001
1002 if (async_error)
1003 goto Complete;
1004
1005 if (pm_wakeup_pending()) {
1006 async_error = -EBUSY;
1007 goto Complete;
1008 }
1009
1010 if (dev->power.syscore)
1011 goto Complete;
1012
1013 dpm_wait_for_children(dev, async);
1014
1015 if (dev->pm_domain) {
1016 info = "noirq power domain ";
1017 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1018 } else if (dev->type && dev->type->pm) {
1019 info = "noirq type ";
1020 callback = pm_noirq_op(dev->type->pm, state);
1021 } else if (dev->class && dev->class->pm) {
1022 info = "noirq class ";
1023 callback = pm_noirq_op(dev->class->pm, state);
1024 } else if (dev->bus && dev->bus->pm) {
1025 info = "noirq bus ";
1026 callback = pm_noirq_op(dev->bus->pm, state);
1027 }
1028
1029 if (!callback && dev->driver && dev->driver->pm) {
1030 info = "noirq driver ";
1031 callback = pm_noirq_op(dev->driver->pm, state);
1032 }
1033
1034 error = dpm_run_callback(callback, dev, state, info);
1035 if (!error)
1036 dev->power.is_noirq_suspended = true;
1037 else
1038 async_error = error;
1039
1040Complete:
1041 complete_all(&dev->power.completion);
1042 return error;
1043}
1044
1045static void async_suspend_noirq(void *data, async_cookie_t cookie)
1046{
1047 struct device *dev = (struct device *)data;
1048 int error;
1049
1050 error = __device_suspend_noirq(dev, pm_transition, true);
1051 if (error) {
1052 dpm_save_failed_dev(dev_name(dev));
1053 pm_dev_err(dev, pm_transition, " async", error);
1054 }
1055
1056 put_device(dev);
1057}
1058
1059static int device_suspend_noirq(struct device *dev)
1060{
1061 reinit_completion(&dev->power.completion);
1062
1063 if (pm_async_enabled && dev->power.async_suspend) {
1064 get_device(dev);
1065 async_schedule(async_suspend_noirq, dev);
1066 return 0;
1067 }
1068 return __device_suspend_noirq(dev, pm_transition, false);
1069}
1070
1071/**
1072 * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1073 * @state: PM transition of the system being carried out.
1074 *
1075 * Prevent device drivers from receiving interrupts and call the "noirq" suspend
1076 * handlers for all non-sysdev devices.
1077 */
1078static int dpm_suspend_noirq(pm_message_t state)
1079{
1080 ktime_t starttime = ktime_get();
1081 int error = 0;
1082
1083 cpuidle_pause();
1084 suspend_device_irqs();
1085 mutex_lock(&dpm_list_mtx);
1086 pm_transition = state;
1087 async_error = 0;
1088
1089 while (!list_empty(&dpm_late_early_list)) {
1090 struct device *dev = to_device(dpm_late_early_list.prev);
1091
1092 get_device(dev);
1093 mutex_unlock(&dpm_list_mtx);
1094
1095 error = device_suspend_noirq(dev);
1096
1097 mutex_lock(&dpm_list_mtx);
1098 if (error) {
1099 pm_dev_err(dev, state, " noirq", error);
1100 dpm_save_failed_dev(dev_name(dev));
1101 put_device(dev);
1102 break;
1103 }
1104 if (!list_empty(&dev->power.entry))
1105 list_move(&dev->power.entry, &dpm_noirq_list);
1106 put_device(dev);
1107
1108 if (async_error)
1109 break;
1110 }
1111 mutex_unlock(&dpm_list_mtx);
1112 async_synchronize_full();
1113 if (!error)
1114 error = async_error;
1115
1116 if (error) {
1117 suspend_stats.failed_suspend_noirq++;
1118 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1119 dpm_resume_noirq(resume_event(state));
1120 } else {
1121 dpm_show_time(starttime, state, "noirq");
1122 }
1123 return error;
1124}
1125
1126/**
1127 * device_suspend_late - Execute a "late suspend" callback for given device.
1128 * @dev: Device to handle.
1129 * @state: PM transition of the system being carried out.
1130 *
1131 * Runtime PM is disabled for @dev while this function is being executed.
1132 */
1133static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1134{
1135 pm_callback_t callback = NULL;
1136 char *info = NULL;
1137 int error = 0;
1138
1139 __pm_runtime_disable(dev, false);
1140
1141 if (async_error)
1142 goto Complete;
1143
1144 if (pm_wakeup_pending()) {
1145 async_error = -EBUSY;
1146 goto Complete;
1147 }
1148
1149 if (dev->power.syscore)
1150 goto Complete;
1151
1152 dpm_wait_for_children(dev, async);
1153
1154 if (dev->pm_domain) {
1155 info = "late power domain ";
1156 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1157 } else if (dev->type && dev->type->pm) {
1158 info = "late type ";
1159 callback = pm_late_early_op(dev->type->pm, state);
1160 } else if (dev->class && dev->class->pm) {
1161 info = "late class ";
1162 callback = pm_late_early_op(dev->class->pm, state);
1163 } else if (dev->bus && dev->bus->pm) {
1164 info = "late bus ";
1165 callback = pm_late_early_op(dev->bus->pm, state);
1166 }
1167
1168 if (!callback && dev->driver && dev->driver->pm) {
1169 info = "late driver ";
1170 callback = pm_late_early_op(dev->driver->pm, state);
1171 }
1172
1173 error = dpm_run_callback(callback, dev, state, info);
1174 if (!error)
1175 dev->power.is_late_suspended = true;
1176 else
1177 async_error = error;
1178
1179Complete:
1180 complete_all(&dev->power.completion);
1181 return error;
1182}
1183
1184static void async_suspend_late(void *data, async_cookie_t cookie)
1185{
1186 struct device *dev = (struct device *)data;
1187 int error;
1188
1189 error = __device_suspend_late(dev, pm_transition, true);
1190 if (error) {
1191 dpm_save_failed_dev(dev_name(dev));
1192 pm_dev_err(dev, pm_transition, " async", error);
1193 }
1194 put_device(dev);
1195}
1196
1197static int device_suspend_late(struct device *dev)
1198{
1199 reinit_completion(&dev->power.completion);
1200
1201 if (pm_async_enabled && dev->power.async_suspend) {
1202 get_device(dev);
1203 async_schedule(async_suspend_late, dev);
1204 return 0;
1205 }
1206
1207 return __device_suspend_late(dev, pm_transition, false);
1208}
1209
1210/**
1211 * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1212 * @state: PM transition of the system being carried out.
1213 */
1214static int dpm_suspend_late(pm_message_t state)
1215{
1216 ktime_t starttime = ktime_get();
1217 int error = 0;
1218
1219 mutex_lock(&dpm_list_mtx);
1220 pm_transition = state;
1221 async_error = 0;
1222
1223 while (!list_empty(&dpm_suspended_list)) {
1224 struct device *dev = to_device(dpm_suspended_list.prev);
1225
1226 get_device(dev);
1227 mutex_unlock(&dpm_list_mtx);
1228
1229 error = device_suspend_late(dev);
1230
1231 mutex_lock(&dpm_list_mtx);
1232 if (error) {
1233 pm_dev_err(dev, state, " late", error);
1234 dpm_save_failed_dev(dev_name(dev));
1235 put_device(dev);
1236 break;
1237 }
1238 if (!list_empty(&dev->power.entry))
1239 list_move(&dev->power.entry, &dpm_late_early_list);
1240 put_device(dev);
1241
1242 if (async_error)
1243 break;
1244 }
1245 mutex_unlock(&dpm_list_mtx);
1246 async_synchronize_full();
1247 if (error) {
1248 suspend_stats.failed_suspend_late++;
1249 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1250 dpm_resume_early(resume_event(state));
1251 } else {
1252 dpm_show_time(starttime, state, "late");
1253 }
1254 return error;
1255}
1256
1257/**
1258 * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1259 * @state: PM transition of the system being carried out.
1260 */
1261int dpm_suspend_end(pm_message_t state)
1262{
1263 int error = dpm_suspend_late(state);
1264 if (error)
1265 return error;
1266
1267 error = dpm_suspend_noirq(state);
1268 if (error) {
1269 dpm_resume_early(resume_event(state));
1270 return error;
1271 }
1272
1273 return 0;
1274}
1275EXPORT_SYMBOL_GPL(dpm_suspend_end);
1276
1277/**
1278 * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1279 * @dev: Device to suspend.
1280 * @state: PM transition of the system being carried out.
1281 * @cb: Suspend callback to execute.
1282 */
1283static int legacy_suspend(struct device *dev, pm_message_t state,
1284 int (*cb)(struct device *dev, pm_message_t state),
1285 char *info)
1286{
1287 int error;
1288 ktime_t calltime;
1289
1290 calltime = initcall_debug_start(dev);
1291
1292 error = cb(dev, state);
1293 suspend_report_result(cb, error);
1294
1295 initcall_debug_report(dev, calltime, error, state, info);
1296
1297 return error;
1298}
1299
1300/**
1301 * device_suspend - Execute "suspend" callbacks for given device.
1302 * @dev: Device to handle.
1303 * @state: PM transition of the system being carried out.
1304 * @async: If true, the device is being suspended asynchronously.
1305 */
1306static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1307{
1308 pm_callback_t callback = NULL;
1309 char *info = NULL;
1310 int error = 0;
1311 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1312
1313 dpm_wait_for_children(dev, async);
1314
1315 if (async_error)
1316 goto Complete;
1317
1318 /*
1319 * If a device configured to wake up the system from sleep states
1320 * has been suspended at run time and there's a resume request pending
1321 * for it, this is equivalent to the device signaling wakeup, so the
1322 * system suspend operation should be aborted.
1323 */
1324 if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
1325 pm_wakeup_event(dev, 0);
1326
1327 if (pm_wakeup_pending()) {
1328 async_error = -EBUSY;
1329 goto Complete;
1330 }
1331
1332 if (dev->power.syscore)
1333 goto Complete;
1334
1335 dpm_watchdog_set(&wd, dev);
1336 device_lock(dev);
1337
1338 if (dev->pm_domain) {
1339 info = "power domain ";
1340 callback = pm_op(&dev->pm_domain->ops, state);
1341 goto Run;
1342 }
1343
1344 if (dev->type && dev->type->pm) {
1345 info = "type ";
1346 callback = pm_op(dev->type->pm, state);
1347 goto Run;
1348 }
1349
1350 if (dev->class) {
1351 if (dev->class->pm) {
1352 info = "class ";
1353 callback = pm_op(dev->class->pm, state);
1354 goto Run;
1355 } else if (dev->class->suspend) {
1356 pm_dev_dbg(dev, state, "legacy class ");
1357 error = legacy_suspend(dev, state, dev->class->suspend,
1358 "legacy class ");
1359 goto End;
1360 }
1361 }
1362
1363 if (dev->bus) {
1364 if (dev->bus->pm) {
1365 info = "bus ";
1366 callback = pm_op(dev->bus->pm, state);
1367 } else if (dev->bus->suspend) {
1368 pm_dev_dbg(dev, state, "legacy bus ");
1369 error = legacy_suspend(dev, state, dev->bus->suspend,
1370 "legacy bus ");
1371 goto End;
1372 }
1373 }
1374
1375 Run:
1376 if (!callback && dev->driver && dev->driver->pm) {
1377 info = "driver ";
1378 callback = pm_op(dev->driver->pm, state);
1379 }
1380
1381 error = dpm_run_callback(callback, dev, state, info);
1382
1383 End:
1384 if (!error) {
1385 dev->power.is_suspended = true;
1386 if (dev->power.wakeup_path
1387 && dev->parent && !dev->parent->power.ignore_children)
1388 dev->parent->power.wakeup_path = true;
1389 }
1390
1391 device_unlock(dev);
1392 dpm_watchdog_clear(&wd);
1393
1394 Complete:
1395 complete_all(&dev->power.completion);
1396 if (error)
1397 async_error = error;
1398
1399 return error;
1400}
1401
1402static void async_suspend(void *data, async_cookie_t cookie)
1403{
1404 struct device *dev = (struct device *)data;
1405 int error;
1406
1407 error = __device_suspend(dev, pm_transition, true);
1408 if (error) {
1409 dpm_save_failed_dev(dev_name(dev));
1410 pm_dev_err(dev, pm_transition, " async", error);
1411 }
1412
1413 put_device(dev);
1414}
1415
1416static int device_suspend(struct device *dev)
1417{
1418 reinit_completion(&dev->power.completion);
1419
1420 if (pm_async_enabled && dev->power.async_suspend) {
1421 get_device(dev);
1422 async_schedule(async_suspend, dev);
1423 return 0;
1424 }
1425
1426 return __device_suspend(dev, pm_transition, false);
1427}
1428
1429/**
1430 * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1431 * @state: PM transition of the system being carried out.
1432 */
1433int dpm_suspend(pm_message_t state)
1434{
1435 ktime_t starttime = ktime_get();
1436 int error = 0;
1437
1438 might_sleep();
1439
1440 cpufreq_suspend();
1441
1442 mutex_lock(&dpm_list_mtx);
1443 pm_transition = state;
1444 async_error = 0;
1445 while (!list_empty(&dpm_prepared_list)) {
1446 struct device *dev = to_device(dpm_prepared_list.prev);
1447
1448 get_device(dev);
1449 mutex_unlock(&dpm_list_mtx);
1450
1451 error = device_suspend(dev);
1452
1453 mutex_lock(&dpm_list_mtx);
1454 if (error) {
1455 pm_dev_err(dev, state, "", error);
1456 dpm_save_failed_dev(dev_name(dev));
1457 put_device(dev);
1458 break;
1459 }
1460 if (!list_empty(&dev->power.entry))
1461 list_move(&dev->power.entry, &dpm_suspended_list);
1462 put_device(dev);
1463 if (async_error)
1464 break;
1465 }
1466 mutex_unlock(&dpm_list_mtx);
1467 async_synchronize_full();
1468 if (!error)
1469 error = async_error;
1470 if (error) {
1471 suspend_stats.failed_suspend++;
1472 dpm_save_failed_step(SUSPEND_SUSPEND);
1473 } else
1474 dpm_show_time(starttime, state, NULL);
1475 return error;
1476}
1477
1478/**
1479 * device_prepare - Prepare a device for system power transition.
1480 * @dev: Device to handle.
1481 * @state: PM transition of the system being carried out.
1482 *
1483 * Execute the ->prepare() callback(s) for given device. No new children of the
1484 * device may be registered after this function has returned.
1485 */
1486static int device_prepare(struct device *dev, pm_message_t state)
1487{
1488 int (*callback)(struct device *) = NULL;
1489 char *info = NULL;
1490 int error = 0;
1491
1492 if (dev->power.syscore)
1493 return 0;
1494
1495 /*
1496 * If a device's parent goes into runtime suspend at the wrong time,
1497 * it won't be possible to resume the device. To prevent this we
1498 * block runtime suspend here, during the prepare phase, and allow
1499 * it again during the complete phase.
1500 */
1501 pm_runtime_get_noresume(dev);
1502
1503 device_lock(dev);
1504
1505 dev->power.wakeup_path = device_may_wakeup(dev);
1506
1507 if (dev->pm_domain) {
1508 info = "preparing power domain ";
1509 callback = dev->pm_domain->ops.prepare;
1510 } else if (dev->type && dev->type->pm) {
1511 info = "preparing type ";
1512 callback = dev->type->pm->prepare;
1513 } else if (dev->class && dev->class->pm) {
1514 info = "preparing class ";
1515 callback = dev->class->pm->prepare;
1516 } else if (dev->bus && dev->bus->pm) {
1517 info = "preparing bus ";
1518 callback = dev->bus->pm->prepare;
1519 }
1520
1521 if (!callback && dev->driver && dev->driver->pm) {
1522 info = "preparing driver ";
1523 callback = dev->driver->pm->prepare;
1524 }
1525
1526 if (callback) {
1527 error = callback(dev);
1528 suspend_report_result(callback, error);
1529 }
1530
1531 device_unlock(dev);
1532
1533 if (error)
1534 pm_runtime_put(dev);
1535
1536 return error;
1537}
1538
1539/**
1540 * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1541 * @state: PM transition of the system being carried out.
1542 *
1543 * Execute the ->prepare() callback(s) for all devices.
1544 */
1545int dpm_prepare(pm_message_t state)
1546{
1547 int error = 0;
1548
1549 might_sleep();
1550
1551 mutex_lock(&dpm_list_mtx);
1552 while (!list_empty(&dpm_list)) {
1553 struct device *dev = to_device(dpm_list.next);
1554
1555 get_device(dev);
1556 mutex_unlock(&dpm_list_mtx);
1557
1558 error = device_prepare(dev, state);
1559
1560 mutex_lock(&dpm_list_mtx);
1561 if (error) {
1562 if (error == -EAGAIN) {
1563 put_device(dev);
1564 error = 0;
1565 continue;
1566 }
1567 printk(KERN_INFO "PM: Device %s not prepared "
1568 "for power transition: code %d\n",
1569 dev_name(dev), error);
1570 put_device(dev);
1571 break;
1572 }
1573 dev->power.is_prepared = true;
1574 if (!list_empty(&dev->power.entry))
1575 list_move_tail(&dev->power.entry, &dpm_prepared_list);
1576 put_device(dev);
1577 }
1578 mutex_unlock(&dpm_list_mtx);
1579 return error;
1580}
1581
1582/**
1583 * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1584 * @state: PM transition of the system being carried out.
1585 *
1586 * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1587 * callbacks for them.
1588 */
1589int dpm_suspend_start(pm_message_t state)
1590{
1591 int error;
1592
1593 error = dpm_prepare(state);
1594 if (error) {
1595 suspend_stats.failed_prepare++;
1596 dpm_save_failed_step(SUSPEND_PREPARE);
1597 } else
1598 error = dpm_suspend(state);
1599 return error;
1600}
1601EXPORT_SYMBOL_GPL(dpm_suspend_start);
1602
1603void __suspend_report_result(const char *function, void *fn, int ret)
1604{
1605 if (ret)
1606 printk(KERN_ERR "%s(): %pF returns %d\n", function, fn, ret);
1607}
1608EXPORT_SYMBOL_GPL(__suspend_report_result);
1609
1610/**
1611 * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1612 * @dev: Device to wait for.
1613 * @subordinate: Device that needs to wait for @dev.
1614 */
1615int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1616{
1617 dpm_wait(dev, subordinate->power.async_suspend);
1618 return async_error;
1619}
1620EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1621
1622/**
1623 * dpm_for_each_dev - device iterator.
1624 * @data: data for the callback.
1625 * @fn: function to be called for each device.
1626 *
1627 * Iterate over devices in dpm_list, and call @fn for each device,
1628 * passing it @data.
1629 */
1630void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1631{
1632 struct device *dev;
1633
1634 if (!fn)
1635 return;
1636
1637 device_pm_lock();
1638 list_for_each_entry(dev, &dpm_list, power.entry)
1639 fn(dev, data);
1640 device_pm_unlock();
1641}
1642EXPORT_SYMBOL_GPL(dpm_for_each_dev);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * drivers/base/power/main.c - Where the driver meets power management.
4 *
5 * Copyright (c) 2003 Patrick Mochel
6 * Copyright (c) 2003 Open Source Development Lab
7 *
8 * The driver model core calls device_pm_add() when a device is registered.
9 * This will initialize the embedded device_pm_info object in the device
10 * and add it to the list of power-controlled devices. sysfs entries for
11 * controlling device power management will also be added.
12 *
13 * A separate list is used for keeping track of power info, because the power
14 * domain dependencies may differ from the ancestral dependencies that the
15 * subsystem list maintains.
16 */
17
18#define pr_fmt(fmt) "PM: " fmt
19#define dev_fmt pr_fmt
20
21#include <linux/device.h>
22#include <linux/export.h>
23#include <linux/mutex.h>
24#include <linux/pm.h>
25#include <linux/pm_runtime.h>
26#include <linux/pm-trace.h>
27#include <linux/pm_wakeirq.h>
28#include <linux/interrupt.h>
29#include <linux/sched.h>
30#include <linux/sched/debug.h>
31#include <linux/async.h>
32#include <linux/suspend.h>
33#include <trace/events/power.h>
34#include <linux/cpufreq.h>
35#include <linux/devfreq.h>
36#include <linux/timer.h>
37
38#include "../base.h"
39#include "power.h"
40
41typedef int (*pm_callback_t)(struct device *);
42
43#define list_for_each_entry_rcu_locked(pos, head, member) \
44 list_for_each_entry_rcu(pos, head, member, \
45 device_links_read_lock_held())
46
47/*
48 * The entries in the dpm_list list are in a depth first order, simply
49 * because children are guaranteed to be discovered after parents, and
50 * are inserted at the back of the list on discovery.
51 *
52 * Since device_pm_add() may be called with a device lock held,
53 * we must never try to acquire a device lock while holding
54 * dpm_list_mutex.
55 */
56
57LIST_HEAD(dpm_list);
58static LIST_HEAD(dpm_prepared_list);
59static LIST_HEAD(dpm_suspended_list);
60static LIST_HEAD(dpm_late_early_list);
61static LIST_HEAD(dpm_noirq_list);
62
63static DEFINE_MUTEX(dpm_list_mtx);
64static pm_message_t pm_transition;
65
66static int async_error;
67
68static const char *pm_verb(int event)
69{
70 switch (event) {
71 case PM_EVENT_SUSPEND:
72 return "suspend";
73 case PM_EVENT_RESUME:
74 return "resume";
75 case PM_EVENT_FREEZE:
76 return "freeze";
77 case PM_EVENT_QUIESCE:
78 return "quiesce";
79 case PM_EVENT_HIBERNATE:
80 return "hibernate";
81 case PM_EVENT_THAW:
82 return "thaw";
83 case PM_EVENT_RESTORE:
84 return "restore";
85 case PM_EVENT_RECOVER:
86 return "recover";
87 default:
88 return "(unknown PM event)";
89 }
90}
91
92/**
93 * device_pm_sleep_init - Initialize system suspend-related device fields.
94 * @dev: Device object being initialized.
95 */
96void device_pm_sleep_init(struct device *dev)
97{
98 dev->power.is_prepared = false;
99 dev->power.is_suspended = false;
100 dev->power.is_noirq_suspended = false;
101 dev->power.is_late_suspended = false;
102 init_completion(&dev->power.completion);
103 complete_all(&dev->power.completion);
104 dev->power.wakeup = NULL;
105 INIT_LIST_HEAD(&dev->power.entry);
106}
107
108/**
109 * device_pm_lock - Lock the list of active devices used by the PM core.
110 */
111void device_pm_lock(void)
112{
113 mutex_lock(&dpm_list_mtx);
114}
115
116/**
117 * device_pm_unlock - Unlock the list of active devices used by the PM core.
118 */
119void device_pm_unlock(void)
120{
121 mutex_unlock(&dpm_list_mtx);
122}
123
124/**
125 * device_pm_add - Add a device to the PM core's list of active devices.
126 * @dev: Device to add to the list.
127 */
128void device_pm_add(struct device *dev)
129{
130 /* Skip PM setup/initialization. */
131 if (device_pm_not_required(dev))
132 return;
133
134 pr_debug("Adding info for %s:%s\n",
135 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
136 device_pm_check_callbacks(dev);
137 mutex_lock(&dpm_list_mtx);
138 if (dev->parent && dev->parent->power.is_prepared)
139 dev_warn(dev, "parent %s should not be sleeping\n",
140 dev_name(dev->parent));
141 list_add_tail(&dev->power.entry, &dpm_list);
142 dev->power.in_dpm_list = true;
143 mutex_unlock(&dpm_list_mtx);
144}
145
146/**
147 * device_pm_remove - Remove a device from the PM core's list of active devices.
148 * @dev: Device to be removed from the list.
149 */
150void device_pm_remove(struct device *dev)
151{
152 if (device_pm_not_required(dev))
153 return;
154
155 pr_debug("Removing info for %s:%s\n",
156 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
157 complete_all(&dev->power.completion);
158 mutex_lock(&dpm_list_mtx);
159 list_del_init(&dev->power.entry);
160 dev->power.in_dpm_list = false;
161 mutex_unlock(&dpm_list_mtx);
162 device_wakeup_disable(dev);
163 pm_runtime_remove(dev);
164 device_pm_check_callbacks(dev);
165}
166
167/**
168 * device_pm_move_before - Move device in the PM core's list of active devices.
169 * @deva: Device to move in dpm_list.
170 * @devb: Device @deva should come before.
171 */
172void device_pm_move_before(struct device *deva, struct device *devb)
173{
174 pr_debug("Moving %s:%s before %s:%s\n",
175 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
176 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
177 /* Delete deva from dpm_list and reinsert before devb. */
178 list_move_tail(&deva->power.entry, &devb->power.entry);
179}
180
181/**
182 * device_pm_move_after - Move device in the PM core's list of active devices.
183 * @deva: Device to move in dpm_list.
184 * @devb: Device @deva should come after.
185 */
186void device_pm_move_after(struct device *deva, struct device *devb)
187{
188 pr_debug("Moving %s:%s after %s:%s\n",
189 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
190 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
191 /* Delete deva from dpm_list and reinsert after devb. */
192 list_move(&deva->power.entry, &devb->power.entry);
193}
194
195/**
196 * device_pm_move_last - Move device to end of the PM core's list of devices.
197 * @dev: Device to move in dpm_list.
198 */
199void device_pm_move_last(struct device *dev)
200{
201 pr_debug("Moving %s:%s to end of list\n",
202 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
203 list_move_tail(&dev->power.entry, &dpm_list);
204}
205
206static ktime_t initcall_debug_start(struct device *dev, void *cb)
207{
208 if (!pm_print_times_enabled)
209 return 0;
210
211 dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
212 task_pid_nr(current),
213 dev->parent ? dev_name(dev->parent) : "none");
214 return ktime_get();
215}
216
217static void initcall_debug_report(struct device *dev, ktime_t calltime,
218 void *cb, int error)
219{
220 ktime_t rettime;
221
222 if (!pm_print_times_enabled)
223 return;
224
225 rettime = ktime_get();
226 dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
227 (unsigned long long)ktime_us_delta(rettime, calltime));
228}
229
230/**
231 * dpm_wait - Wait for a PM operation to complete.
232 * @dev: Device to wait for.
233 * @async: If unset, wait only if the device's power.async_suspend flag is set.
234 */
235static void dpm_wait(struct device *dev, bool async)
236{
237 if (!dev)
238 return;
239
240 if (async || (pm_async_enabled && dev->power.async_suspend))
241 wait_for_completion(&dev->power.completion);
242}
243
244static int dpm_wait_fn(struct device *dev, void *async_ptr)
245{
246 dpm_wait(dev, *((bool *)async_ptr));
247 return 0;
248}
249
250static void dpm_wait_for_children(struct device *dev, bool async)
251{
252 device_for_each_child(dev, &async, dpm_wait_fn);
253}
254
255static void dpm_wait_for_suppliers(struct device *dev, bool async)
256{
257 struct device_link *link;
258 int idx;
259
260 idx = device_links_read_lock();
261
262 /*
263 * If the supplier goes away right after we've checked the link to it,
264 * we'll wait for its completion to change the state, but that's fine,
265 * because the only things that will block as a result are the SRCU
266 * callbacks freeing the link objects for the links in the list we're
267 * walking.
268 */
269 list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
270 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
271 dpm_wait(link->supplier, async);
272
273 device_links_read_unlock(idx);
274}
275
276static bool dpm_wait_for_superior(struct device *dev, bool async)
277{
278 struct device *parent;
279
280 /*
281 * If the device is resumed asynchronously and the parent's callback
282 * deletes both the device and the parent itself, the parent object may
283 * be freed while this function is running, so avoid that by reference
284 * counting the parent once more unless the device has been deleted
285 * already (in which case return right away).
286 */
287 mutex_lock(&dpm_list_mtx);
288
289 if (!device_pm_initialized(dev)) {
290 mutex_unlock(&dpm_list_mtx);
291 return false;
292 }
293
294 parent = get_device(dev->parent);
295
296 mutex_unlock(&dpm_list_mtx);
297
298 dpm_wait(parent, async);
299 put_device(parent);
300
301 dpm_wait_for_suppliers(dev, async);
302
303 /*
304 * If the parent's callback has deleted the device, attempting to resume
305 * it would be invalid, so avoid doing that then.
306 */
307 return device_pm_initialized(dev);
308}
309
310static void dpm_wait_for_consumers(struct device *dev, bool async)
311{
312 struct device_link *link;
313 int idx;
314
315 idx = device_links_read_lock();
316
317 /*
318 * The status of a device link can only be changed from "dormant" by a
319 * probe, but that cannot happen during system suspend/resume. In
320 * theory it can change to "dormant" at that time, but then it is
321 * reasonable to wait for the target device anyway (eg. if it goes
322 * away, it's better to wait for it to go away completely and then
323 * continue instead of trying to continue in parallel with its
324 * unregistration).
325 */
326 list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
327 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
328 dpm_wait(link->consumer, async);
329
330 device_links_read_unlock(idx);
331}
332
333static void dpm_wait_for_subordinate(struct device *dev, bool async)
334{
335 dpm_wait_for_children(dev, async);
336 dpm_wait_for_consumers(dev, async);
337}
338
339/**
340 * pm_op - Return the PM operation appropriate for given PM event.
341 * @ops: PM operations to choose from.
342 * @state: PM transition of the system being carried out.
343 */
344static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
345{
346 switch (state.event) {
347#ifdef CONFIG_SUSPEND
348 case PM_EVENT_SUSPEND:
349 return ops->suspend;
350 case PM_EVENT_RESUME:
351 return ops->resume;
352#endif /* CONFIG_SUSPEND */
353#ifdef CONFIG_HIBERNATE_CALLBACKS
354 case PM_EVENT_FREEZE:
355 case PM_EVENT_QUIESCE:
356 return ops->freeze;
357 case PM_EVENT_HIBERNATE:
358 return ops->poweroff;
359 case PM_EVENT_THAW:
360 case PM_EVENT_RECOVER:
361 return ops->thaw;
362 case PM_EVENT_RESTORE:
363 return ops->restore;
364#endif /* CONFIG_HIBERNATE_CALLBACKS */
365 }
366
367 return NULL;
368}
369
370/**
371 * pm_late_early_op - Return the PM operation appropriate for given PM event.
372 * @ops: PM operations to choose from.
373 * @state: PM transition of the system being carried out.
374 *
375 * Runtime PM is disabled for @dev while this function is being executed.
376 */
377static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
378 pm_message_t state)
379{
380 switch (state.event) {
381#ifdef CONFIG_SUSPEND
382 case PM_EVENT_SUSPEND:
383 return ops->suspend_late;
384 case PM_EVENT_RESUME:
385 return ops->resume_early;
386#endif /* CONFIG_SUSPEND */
387#ifdef CONFIG_HIBERNATE_CALLBACKS
388 case PM_EVENT_FREEZE:
389 case PM_EVENT_QUIESCE:
390 return ops->freeze_late;
391 case PM_EVENT_HIBERNATE:
392 return ops->poweroff_late;
393 case PM_EVENT_THAW:
394 case PM_EVENT_RECOVER:
395 return ops->thaw_early;
396 case PM_EVENT_RESTORE:
397 return ops->restore_early;
398#endif /* CONFIG_HIBERNATE_CALLBACKS */
399 }
400
401 return NULL;
402}
403
404/**
405 * pm_noirq_op - Return the PM operation appropriate for given PM event.
406 * @ops: PM operations to choose from.
407 * @state: PM transition of the system being carried out.
408 *
409 * The driver of @dev will not receive interrupts while this function is being
410 * executed.
411 */
412static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
413{
414 switch (state.event) {
415#ifdef CONFIG_SUSPEND
416 case PM_EVENT_SUSPEND:
417 return ops->suspend_noirq;
418 case PM_EVENT_RESUME:
419 return ops->resume_noirq;
420#endif /* CONFIG_SUSPEND */
421#ifdef CONFIG_HIBERNATE_CALLBACKS
422 case PM_EVENT_FREEZE:
423 case PM_EVENT_QUIESCE:
424 return ops->freeze_noirq;
425 case PM_EVENT_HIBERNATE:
426 return ops->poweroff_noirq;
427 case PM_EVENT_THAW:
428 case PM_EVENT_RECOVER:
429 return ops->thaw_noirq;
430 case PM_EVENT_RESTORE:
431 return ops->restore_noirq;
432#endif /* CONFIG_HIBERNATE_CALLBACKS */
433 }
434
435 return NULL;
436}
437
438static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
439{
440 dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
441 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
442 ", may wakeup" : "", dev->power.driver_flags);
443}
444
445static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
446 int error)
447{
448 dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
449 error);
450}
451
452static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
453 const char *info)
454{
455 ktime_t calltime;
456 u64 usecs64;
457 int usecs;
458
459 calltime = ktime_get();
460 usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
461 do_div(usecs64, NSEC_PER_USEC);
462 usecs = usecs64;
463 if (usecs == 0)
464 usecs = 1;
465
466 pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
467 info ?: "", info ? " " : "", pm_verb(state.event),
468 error ? "aborted" : "complete",
469 usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
470}
471
472static int dpm_run_callback(pm_callback_t cb, struct device *dev,
473 pm_message_t state, const char *info)
474{
475 ktime_t calltime;
476 int error;
477
478 if (!cb)
479 return 0;
480
481 calltime = initcall_debug_start(dev, cb);
482
483 pm_dev_dbg(dev, state, info);
484 trace_device_pm_callback_start(dev, info, state.event);
485 error = cb(dev);
486 trace_device_pm_callback_end(dev, error);
487 suspend_report_result(dev, cb, error);
488
489 initcall_debug_report(dev, calltime, cb, error);
490
491 return error;
492}
493
494#ifdef CONFIG_DPM_WATCHDOG
495struct dpm_watchdog {
496 struct device *dev;
497 struct task_struct *tsk;
498 struct timer_list timer;
499};
500
501#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
502 struct dpm_watchdog wd
503
504/**
505 * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
506 * @t: The timer that PM watchdog depends on.
507 *
508 * Called when a driver has timed out suspending or resuming.
509 * There's not much we can do here to recover so panic() to
510 * capture a crash-dump in pstore.
511 */
512static void dpm_watchdog_handler(struct timer_list *t)
513{
514 struct dpm_watchdog *wd = from_timer(wd, t, timer);
515
516 dev_emerg(wd->dev, "**** DPM device timeout ****\n");
517 show_stack(wd->tsk, NULL, KERN_EMERG);
518 panic("%s %s: unrecoverable failure\n",
519 dev_driver_string(wd->dev), dev_name(wd->dev));
520}
521
522/**
523 * dpm_watchdog_set - Enable pm watchdog for given device.
524 * @wd: Watchdog. Must be allocated on the stack.
525 * @dev: Device to handle.
526 */
527static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
528{
529 struct timer_list *timer = &wd->timer;
530
531 wd->dev = dev;
532 wd->tsk = current;
533
534 timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
535 /* use same timeout value for both suspend and resume */
536 timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
537 add_timer(timer);
538}
539
540/**
541 * dpm_watchdog_clear - Disable suspend/resume watchdog.
542 * @wd: Watchdog to disable.
543 */
544static void dpm_watchdog_clear(struct dpm_watchdog *wd)
545{
546 struct timer_list *timer = &wd->timer;
547
548 del_timer_sync(timer);
549 destroy_timer_on_stack(timer);
550}
551#else
552#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
553#define dpm_watchdog_set(x, y)
554#define dpm_watchdog_clear(x)
555#endif
556
557/*------------------------- Resume routines -------------------------*/
558
559/**
560 * dev_pm_skip_resume - System-wide device resume optimization check.
561 * @dev: Target device.
562 *
563 * Return:
564 * - %false if the transition under way is RESTORE.
565 * - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
566 * - The logical negation of %power.must_resume otherwise (that is, when the
567 * transition under way is RESUME).
568 */
569bool dev_pm_skip_resume(struct device *dev)
570{
571 if (pm_transition.event == PM_EVENT_RESTORE)
572 return false;
573
574 if (pm_transition.event == PM_EVENT_THAW)
575 return dev_pm_skip_suspend(dev);
576
577 return !dev->power.must_resume;
578}
579
580static bool is_async(struct device *dev)
581{
582 return dev->power.async_suspend && pm_async_enabled
583 && !pm_trace_is_enabled();
584}
585
586static bool dpm_async_fn(struct device *dev, async_func_t func)
587{
588 reinit_completion(&dev->power.completion);
589
590 if (is_async(dev)) {
591 dev->power.async_in_progress = true;
592
593 get_device(dev);
594
595 if (async_schedule_dev_nocall(func, dev))
596 return true;
597
598 put_device(dev);
599 }
600 /*
601 * Because async_schedule_dev_nocall() above has returned false or it
602 * has not been called at all, func() is not running and it is safe to
603 * update the async_in_progress flag without extra synchronization.
604 */
605 dev->power.async_in_progress = false;
606 return false;
607}
608
609/**
610 * device_resume_noirq - Execute a "noirq resume" callback for given device.
611 * @dev: Device to handle.
612 * @state: PM transition of the system being carried out.
613 * @async: If true, the device is being resumed asynchronously.
614 *
615 * The driver of @dev will not receive interrupts while this function is being
616 * executed.
617 */
618static void device_resume_noirq(struct device *dev, pm_message_t state, bool async)
619{
620 pm_callback_t callback = NULL;
621 const char *info = NULL;
622 bool skip_resume;
623 int error = 0;
624
625 TRACE_DEVICE(dev);
626 TRACE_RESUME(0);
627
628 if (dev->power.syscore || dev->power.direct_complete)
629 goto Out;
630
631 if (!dev->power.is_noirq_suspended)
632 goto Out;
633
634 if (!dpm_wait_for_superior(dev, async))
635 goto Out;
636
637 skip_resume = dev_pm_skip_resume(dev);
638 /*
639 * If the driver callback is skipped below or by the middle layer
640 * callback and device_resume_early() also skips the driver callback for
641 * this device later, it needs to appear as "suspended" to PM-runtime,
642 * so change its status accordingly.
643 *
644 * Otherwise, the device is going to be resumed, so set its PM-runtime
645 * status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set
646 * to avoid confusing drivers that don't use it.
647 */
648 if (skip_resume)
649 pm_runtime_set_suspended(dev);
650 else if (dev_pm_skip_suspend(dev))
651 pm_runtime_set_active(dev);
652
653 if (dev->pm_domain) {
654 info = "noirq power domain ";
655 callback = pm_noirq_op(&dev->pm_domain->ops, state);
656 } else if (dev->type && dev->type->pm) {
657 info = "noirq type ";
658 callback = pm_noirq_op(dev->type->pm, state);
659 } else if (dev->class && dev->class->pm) {
660 info = "noirq class ";
661 callback = pm_noirq_op(dev->class->pm, state);
662 } else if (dev->bus && dev->bus->pm) {
663 info = "noirq bus ";
664 callback = pm_noirq_op(dev->bus->pm, state);
665 }
666 if (callback)
667 goto Run;
668
669 if (skip_resume)
670 goto Skip;
671
672 if (dev->driver && dev->driver->pm) {
673 info = "noirq driver ";
674 callback = pm_noirq_op(dev->driver->pm, state);
675 }
676
677Run:
678 error = dpm_run_callback(callback, dev, state, info);
679
680Skip:
681 dev->power.is_noirq_suspended = false;
682
683Out:
684 complete_all(&dev->power.completion);
685 TRACE_RESUME(error);
686
687 if (error) {
688 async_error = error;
689 dpm_save_failed_dev(dev_name(dev));
690 pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
691 }
692}
693
694static void async_resume_noirq(void *data, async_cookie_t cookie)
695{
696 struct device *dev = data;
697
698 device_resume_noirq(dev, pm_transition, true);
699 put_device(dev);
700}
701
702static void dpm_noirq_resume_devices(pm_message_t state)
703{
704 struct device *dev;
705 ktime_t starttime = ktime_get();
706
707 trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
708
709 async_error = 0;
710 pm_transition = state;
711
712 mutex_lock(&dpm_list_mtx);
713
714 /*
715 * Trigger the resume of "async" devices upfront so they don't have to
716 * wait for the "non-async" ones they don't depend on.
717 */
718 list_for_each_entry(dev, &dpm_noirq_list, power.entry)
719 dpm_async_fn(dev, async_resume_noirq);
720
721 while (!list_empty(&dpm_noirq_list)) {
722 dev = to_device(dpm_noirq_list.next);
723 list_move_tail(&dev->power.entry, &dpm_late_early_list);
724
725 if (!dev->power.async_in_progress) {
726 get_device(dev);
727
728 mutex_unlock(&dpm_list_mtx);
729
730 device_resume_noirq(dev, state, false);
731
732 put_device(dev);
733
734 mutex_lock(&dpm_list_mtx);
735 }
736 }
737 mutex_unlock(&dpm_list_mtx);
738 async_synchronize_full();
739 dpm_show_time(starttime, state, 0, "noirq");
740 if (async_error)
741 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
742
743 trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
744}
745
746/**
747 * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
748 * @state: PM transition of the system being carried out.
749 *
750 * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
751 * allow device drivers' interrupt handlers to be called.
752 */
753void dpm_resume_noirq(pm_message_t state)
754{
755 dpm_noirq_resume_devices(state);
756
757 resume_device_irqs();
758 device_wakeup_disarm_wake_irqs();
759}
760
761/**
762 * device_resume_early - Execute an "early resume" callback for given device.
763 * @dev: Device to handle.
764 * @state: PM transition of the system being carried out.
765 * @async: If true, the device is being resumed asynchronously.
766 *
767 * Runtime PM is disabled for @dev while this function is being executed.
768 */
769static void device_resume_early(struct device *dev, pm_message_t state, bool async)
770{
771 pm_callback_t callback = NULL;
772 const char *info = NULL;
773 int error = 0;
774
775 TRACE_DEVICE(dev);
776 TRACE_RESUME(0);
777
778 if (dev->power.syscore || dev->power.direct_complete)
779 goto Out;
780
781 if (!dev->power.is_late_suspended)
782 goto Out;
783
784 if (!dpm_wait_for_superior(dev, async))
785 goto Out;
786
787 if (dev->pm_domain) {
788 info = "early power domain ";
789 callback = pm_late_early_op(&dev->pm_domain->ops, state);
790 } else if (dev->type && dev->type->pm) {
791 info = "early type ";
792 callback = pm_late_early_op(dev->type->pm, state);
793 } else if (dev->class && dev->class->pm) {
794 info = "early class ";
795 callback = pm_late_early_op(dev->class->pm, state);
796 } else if (dev->bus && dev->bus->pm) {
797 info = "early bus ";
798 callback = pm_late_early_op(dev->bus->pm, state);
799 }
800 if (callback)
801 goto Run;
802
803 if (dev_pm_skip_resume(dev))
804 goto Skip;
805
806 if (dev->driver && dev->driver->pm) {
807 info = "early driver ";
808 callback = pm_late_early_op(dev->driver->pm, state);
809 }
810
811Run:
812 error = dpm_run_callback(callback, dev, state, info);
813
814Skip:
815 dev->power.is_late_suspended = false;
816
817Out:
818 TRACE_RESUME(error);
819
820 pm_runtime_enable(dev);
821 complete_all(&dev->power.completion);
822
823 if (error) {
824 async_error = error;
825 dpm_save_failed_dev(dev_name(dev));
826 pm_dev_err(dev, state, async ? " async early" : " early", error);
827 }
828}
829
830static void async_resume_early(void *data, async_cookie_t cookie)
831{
832 struct device *dev = data;
833
834 device_resume_early(dev, pm_transition, true);
835 put_device(dev);
836}
837
838/**
839 * dpm_resume_early - Execute "early resume" callbacks for all devices.
840 * @state: PM transition of the system being carried out.
841 */
842void dpm_resume_early(pm_message_t state)
843{
844 struct device *dev;
845 ktime_t starttime = ktime_get();
846
847 trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
848
849 async_error = 0;
850 pm_transition = state;
851
852 mutex_lock(&dpm_list_mtx);
853
854 /*
855 * Trigger the resume of "async" devices upfront so they don't have to
856 * wait for the "non-async" ones they don't depend on.
857 */
858 list_for_each_entry(dev, &dpm_late_early_list, power.entry)
859 dpm_async_fn(dev, async_resume_early);
860
861 while (!list_empty(&dpm_late_early_list)) {
862 dev = to_device(dpm_late_early_list.next);
863 list_move_tail(&dev->power.entry, &dpm_suspended_list);
864
865 if (!dev->power.async_in_progress) {
866 get_device(dev);
867
868 mutex_unlock(&dpm_list_mtx);
869
870 device_resume_early(dev, state, false);
871
872 put_device(dev);
873
874 mutex_lock(&dpm_list_mtx);
875 }
876 }
877 mutex_unlock(&dpm_list_mtx);
878 async_synchronize_full();
879 dpm_show_time(starttime, state, 0, "early");
880 if (async_error)
881 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
882
883 trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
884}
885
886/**
887 * dpm_resume_start - Execute "noirq" and "early" device callbacks.
888 * @state: PM transition of the system being carried out.
889 */
890void dpm_resume_start(pm_message_t state)
891{
892 dpm_resume_noirq(state);
893 dpm_resume_early(state);
894}
895EXPORT_SYMBOL_GPL(dpm_resume_start);
896
897/**
898 * device_resume - Execute "resume" callbacks for given device.
899 * @dev: Device to handle.
900 * @state: PM transition of the system being carried out.
901 * @async: If true, the device is being resumed asynchronously.
902 */
903static void device_resume(struct device *dev, pm_message_t state, bool async)
904{
905 pm_callback_t callback = NULL;
906 const char *info = NULL;
907 int error = 0;
908 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
909
910 TRACE_DEVICE(dev);
911 TRACE_RESUME(0);
912
913 if (dev->power.syscore)
914 goto Complete;
915
916 if (dev->power.direct_complete) {
917 /* Match the pm_runtime_disable() in __device_suspend(). */
918 pm_runtime_enable(dev);
919 goto Complete;
920 }
921
922 if (!dpm_wait_for_superior(dev, async))
923 goto Complete;
924
925 dpm_watchdog_set(&wd, dev);
926 device_lock(dev);
927
928 /*
929 * This is a fib. But we'll allow new children to be added below
930 * a resumed device, even if the device hasn't been completed yet.
931 */
932 dev->power.is_prepared = false;
933
934 if (!dev->power.is_suspended)
935 goto Unlock;
936
937 if (dev->pm_domain) {
938 info = "power domain ";
939 callback = pm_op(&dev->pm_domain->ops, state);
940 goto Driver;
941 }
942
943 if (dev->type && dev->type->pm) {
944 info = "type ";
945 callback = pm_op(dev->type->pm, state);
946 goto Driver;
947 }
948
949 if (dev->class && dev->class->pm) {
950 info = "class ";
951 callback = pm_op(dev->class->pm, state);
952 goto Driver;
953 }
954
955 if (dev->bus) {
956 if (dev->bus->pm) {
957 info = "bus ";
958 callback = pm_op(dev->bus->pm, state);
959 } else if (dev->bus->resume) {
960 info = "legacy bus ";
961 callback = dev->bus->resume;
962 goto End;
963 }
964 }
965
966 Driver:
967 if (!callback && dev->driver && dev->driver->pm) {
968 info = "driver ";
969 callback = pm_op(dev->driver->pm, state);
970 }
971
972 End:
973 error = dpm_run_callback(callback, dev, state, info);
974 dev->power.is_suspended = false;
975
976 Unlock:
977 device_unlock(dev);
978 dpm_watchdog_clear(&wd);
979
980 Complete:
981 complete_all(&dev->power.completion);
982
983 TRACE_RESUME(error);
984
985 if (error) {
986 async_error = error;
987 dpm_save_failed_dev(dev_name(dev));
988 pm_dev_err(dev, state, async ? " async" : "", error);
989 }
990}
991
992static void async_resume(void *data, async_cookie_t cookie)
993{
994 struct device *dev = data;
995
996 device_resume(dev, pm_transition, true);
997 put_device(dev);
998}
999
1000/**
1001 * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
1002 * @state: PM transition of the system being carried out.
1003 *
1004 * Execute the appropriate "resume" callback for all devices whose status
1005 * indicates that they are suspended.
1006 */
1007void dpm_resume(pm_message_t state)
1008{
1009 struct device *dev;
1010 ktime_t starttime = ktime_get();
1011
1012 trace_suspend_resume(TPS("dpm_resume"), state.event, true);
1013 might_sleep();
1014
1015 pm_transition = state;
1016 async_error = 0;
1017
1018 mutex_lock(&dpm_list_mtx);
1019
1020 /*
1021 * Trigger the resume of "async" devices upfront so they don't have to
1022 * wait for the "non-async" ones they don't depend on.
1023 */
1024 list_for_each_entry(dev, &dpm_suspended_list, power.entry)
1025 dpm_async_fn(dev, async_resume);
1026
1027 while (!list_empty(&dpm_suspended_list)) {
1028 dev = to_device(dpm_suspended_list.next);
1029 list_move_tail(&dev->power.entry, &dpm_prepared_list);
1030
1031 if (!dev->power.async_in_progress) {
1032 get_device(dev);
1033
1034 mutex_unlock(&dpm_list_mtx);
1035
1036 device_resume(dev, state, false);
1037
1038 put_device(dev);
1039
1040 mutex_lock(&dpm_list_mtx);
1041 }
1042 }
1043 mutex_unlock(&dpm_list_mtx);
1044 async_synchronize_full();
1045 dpm_show_time(starttime, state, 0, NULL);
1046 if (async_error)
1047 dpm_save_failed_step(SUSPEND_RESUME);
1048
1049 cpufreq_resume();
1050 devfreq_resume();
1051 trace_suspend_resume(TPS("dpm_resume"), state.event, false);
1052}
1053
1054/**
1055 * device_complete - Complete a PM transition for given device.
1056 * @dev: Device to handle.
1057 * @state: PM transition of the system being carried out.
1058 */
1059static void device_complete(struct device *dev, pm_message_t state)
1060{
1061 void (*callback)(struct device *) = NULL;
1062 const char *info = NULL;
1063
1064 if (dev->power.syscore)
1065 goto out;
1066
1067 device_lock(dev);
1068
1069 if (dev->pm_domain) {
1070 info = "completing power domain ";
1071 callback = dev->pm_domain->ops.complete;
1072 } else if (dev->type && dev->type->pm) {
1073 info = "completing type ";
1074 callback = dev->type->pm->complete;
1075 } else if (dev->class && dev->class->pm) {
1076 info = "completing class ";
1077 callback = dev->class->pm->complete;
1078 } else if (dev->bus && dev->bus->pm) {
1079 info = "completing bus ";
1080 callback = dev->bus->pm->complete;
1081 }
1082
1083 if (!callback && dev->driver && dev->driver->pm) {
1084 info = "completing driver ";
1085 callback = dev->driver->pm->complete;
1086 }
1087
1088 if (callback) {
1089 pm_dev_dbg(dev, state, info);
1090 callback(dev);
1091 }
1092
1093 device_unlock(dev);
1094
1095out:
1096 pm_runtime_put(dev);
1097}
1098
1099/**
1100 * dpm_complete - Complete a PM transition for all non-sysdev devices.
1101 * @state: PM transition of the system being carried out.
1102 *
1103 * Execute the ->complete() callbacks for all devices whose PM status is not
1104 * DPM_ON (this allows new devices to be registered).
1105 */
1106void dpm_complete(pm_message_t state)
1107{
1108 struct list_head list;
1109
1110 trace_suspend_resume(TPS("dpm_complete"), state.event, true);
1111 might_sleep();
1112
1113 INIT_LIST_HEAD(&list);
1114 mutex_lock(&dpm_list_mtx);
1115 while (!list_empty(&dpm_prepared_list)) {
1116 struct device *dev = to_device(dpm_prepared_list.prev);
1117
1118 get_device(dev);
1119 dev->power.is_prepared = false;
1120 list_move(&dev->power.entry, &list);
1121
1122 mutex_unlock(&dpm_list_mtx);
1123
1124 trace_device_pm_callback_start(dev, "", state.event);
1125 device_complete(dev, state);
1126 trace_device_pm_callback_end(dev, 0);
1127
1128 put_device(dev);
1129
1130 mutex_lock(&dpm_list_mtx);
1131 }
1132 list_splice(&list, &dpm_list);
1133 mutex_unlock(&dpm_list_mtx);
1134
1135 /* Allow device probing and trigger re-probing of deferred devices */
1136 device_unblock_probing();
1137 trace_suspend_resume(TPS("dpm_complete"), state.event, false);
1138}
1139
1140/**
1141 * dpm_resume_end - Execute "resume" callbacks and complete system transition.
1142 * @state: PM transition of the system being carried out.
1143 *
1144 * Execute "resume" callbacks for all devices and complete the PM transition of
1145 * the system.
1146 */
1147void dpm_resume_end(pm_message_t state)
1148{
1149 dpm_resume(state);
1150 dpm_complete(state);
1151}
1152EXPORT_SYMBOL_GPL(dpm_resume_end);
1153
1154
1155/*------------------------- Suspend routines -------------------------*/
1156
1157/**
1158 * resume_event - Return a "resume" message for given "suspend" sleep state.
1159 * @sleep_state: PM message representing a sleep state.
1160 *
1161 * Return a PM message representing the resume event corresponding to given
1162 * sleep state.
1163 */
1164static pm_message_t resume_event(pm_message_t sleep_state)
1165{
1166 switch (sleep_state.event) {
1167 case PM_EVENT_SUSPEND:
1168 return PMSG_RESUME;
1169 case PM_EVENT_FREEZE:
1170 case PM_EVENT_QUIESCE:
1171 return PMSG_RECOVER;
1172 case PM_EVENT_HIBERNATE:
1173 return PMSG_RESTORE;
1174 }
1175 return PMSG_ON;
1176}
1177
1178static void dpm_superior_set_must_resume(struct device *dev)
1179{
1180 struct device_link *link;
1181 int idx;
1182
1183 if (dev->parent)
1184 dev->parent->power.must_resume = true;
1185
1186 idx = device_links_read_lock();
1187
1188 list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
1189 link->supplier->power.must_resume = true;
1190
1191 device_links_read_unlock(idx);
1192}
1193
1194/**
1195 * device_suspend_noirq - Execute a "noirq suspend" callback for given device.
1196 * @dev: Device to handle.
1197 * @state: PM transition of the system being carried out.
1198 * @async: If true, the device is being suspended asynchronously.
1199 *
1200 * The driver of @dev will not receive interrupts while this function is being
1201 * executed.
1202 */
1203static int device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1204{
1205 pm_callback_t callback = NULL;
1206 const char *info = NULL;
1207 int error = 0;
1208
1209 TRACE_DEVICE(dev);
1210 TRACE_SUSPEND(0);
1211
1212 dpm_wait_for_subordinate(dev, async);
1213
1214 if (async_error)
1215 goto Complete;
1216
1217 if (dev->power.syscore || dev->power.direct_complete)
1218 goto Complete;
1219
1220 if (dev->pm_domain) {
1221 info = "noirq power domain ";
1222 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1223 } else if (dev->type && dev->type->pm) {
1224 info = "noirq type ";
1225 callback = pm_noirq_op(dev->type->pm, state);
1226 } else if (dev->class && dev->class->pm) {
1227 info = "noirq class ";
1228 callback = pm_noirq_op(dev->class->pm, state);
1229 } else if (dev->bus && dev->bus->pm) {
1230 info = "noirq bus ";
1231 callback = pm_noirq_op(dev->bus->pm, state);
1232 }
1233 if (callback)
1234 goto Run;
1235
1236 if (dev_pm_skip_suspend(dev))
1237 goto Skip;
1238
1239 if (dev->driver && dev->driver->pm) {
1240 info = "noirq driver ";
1241 callback = pm_noirq_op(dev->driver->pm, state);
1242 }
1243
1244Run:
1245 error = dpm_run_callback(callback, dev, state, info);
1246 if (error) {
1247 async_error = error;
1248 dpm_save_failed_dev(dev_name(dev));
1249 pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
1250 goto Complete;
1251 }
1252
1253Skip:
1254 dev->power.is_noirq_suspended = true;
1255
1256 /*
1257 * Skipping the resume of devices that were in use right before the
1258 * system suspend (as indicated by their PM-runtime usage counters)
1259 * would be suboptimal. Also resume them if doing that is not allowed
1260 * to be skipped.
1261 */
1262 if (atomic_read(&dev->power.usage_count) > 1 ||
1263 !(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
1264 dev->power.may_skip_resume))
1265 dev->power.must_resume = true;
1266
1267 if (dev->power.must_resume)
1268 dpm_superior_set_must_resume(dev);
1269
1270Complete:
1271 complete_all(&dev->power.completion);
1272 TRACE_SUSPEND(error);
1273 return error;
1274}
1275
1276static void async_suspend_noirq(void *data, async_cookie_t cookie)
1277{
1278 struct device *dev = data;
1279
1280 device_suspend_noirq(dev, pm_transition, true);
1281 put_device(dev);
1282}
1283
1284static int dpm_noirq_suspend_devices(pm_message_t state)
1285{
1286 ktime_t starttime = ktime_get();
1287 int error = 0;
1288
1289 trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1290
1291 pm_transition = state;
1292 async_error = 0;
1293
1294 mutex_lock(&dpm_list_mtx);
1295
1296 while (!list_empty(&dpm_late_early_list)) {
1297 struct device *dev = to_device(dpm_late_early_list.prev);
1298
1299 list_move(&dev->power.entry, &dpm_noirq_list);
1300
1301 if (dpm_async_fn(dev, async_suspend_noirq))
1302 continue;
1303
1304 get_device(dev);
1305
1306 mutex_unlock(&dpm_list_mtx);
1307
1308 error = device_suspend_noirq(dev, state, false);
1309
1310 put_device(dev);
1311
1312 mutex_lock(&dpm_list_mtx);
1313
1314 if (error || async_error)
1315 break;
1316 }
1317
1318 mutex_unlock(&dpm_list_mtx);
1319
1320 async_synchronize_full();
1321 if (!error)
1322 error = async_error;
1323
1324 if (error)
1325 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1326
1327 dpm_show_time(starttime, state, error, "noirq");
1328 trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1329 return error;
1330}
1331
1332/**
1333 * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1334 * @state: PM transition of the system being carried out.
1335 *
1336 * Prevent device drivers' interrupt handlers from being called and invoke
1337 * "noirq" suspend callbacks for all non-sysdev devices.
1338 */
1339int dpm_suspend_noirq(pm_message_t state)
1340{
1341 int ret;
1342
1343 device_wakeup_arm_wake_irqs();
1344 suspend_device_irqs();
1345
1346 ret = dpm_noirq_suspend_devices(state);
1347 if (ret)
1348 dpm_resume_noirq(resume_event(state));
1349
1350 return ret;
1351}
1352
1353static void dpm_propagate_wakeup_to_parent(struct device *dev)
1354{
1355 struct device *parent = dev->parent;
1356
1357 if (!parent)
1358 return;
1359
1360 spin_lock_irq(&parent->power.lock);
1361
1362 if (device_wakeup_path(dev) && !parent->power.ignore_children)
1363 parent->power.wakeup_path = true;
1364
1365 spin_unlock_irq(&parent->power.lock);
1366}
1367
1368/**
1369 * device_suspend_late - Execute a "late suspend" callback for given device.
1370 * @dev: Device to handle.
1371 * @state: PM transition of the system being carried out.
1372 * @async: If true, the device is being suspended asynchronously.
1373 *
1374 * Runtime PM is disabled for @dev while this function is being executed.
1375 */
1376static int device_suspend_late(struct device *dev, pm_message_t state, bool async)
1377{
1378 pm_callback_t callback = NULL;
1379 const char *info = NULL;
1380 int error = 0;
1381
1382 TRACE_DEVICE(dev);
1383 TRACE_SUSPEND(0);
1384
1385 __pm_runtime_disable(dev, false);
1386
1387 dpm_wait_for_subordinate(dev, async);
1388
1389 if (async_error)
1390 goto Complete;
1391
1392 if (pm_wakeup_pending()) {
1393 async_error = -EBUSY;
1394 goto Complete;
1395 }
1396
1397 if (dev->power.syscore || dev->power.direct_complete)
1398 goto Complete;
1399
1400 if (dev->pm_domain) {
1401 info = "late power domain ";
1402 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1403 } else if (dev->type && dev->type->pm) {
1404 info = "late type ";
1405 callback = pm_late_early_op(dev->type->pm, state);
1406 } else if (dev->class && dev->class->pm) {
1407 info = "late class ";
1408 callback = pm_late_early_op(dev->class->pm, state);
1409 } else if (dev->bus && dev->bus->pm) {
1410 info = "late bus ";
1411 callback = pm_late_early_op(dev->bus->pm, state);
1412 }
1413 if (callback)
1414 goto Run;
1415
1416 if (dev_pm_skip_suspend(dev))
1417 goto Skip;
1418
1419 if (dev->driver && dev->driver->pm) {
1420 info = "late driver ";
1421 callback = pm_late_early_op(dev->driver->pm, state);
1422 }
1423
1424Run:
1425 error = dpm_run_callback(callback, dev, state, info);
1426 if (error) {
1427 async_error = error;
1428 dpm_save_failed_dev(dev_name(dev));
1429 pm_dev_err(dev, state, async ? " async late" : " late", error);
1430 goto Complete;
1431 }
1432 dpm_propagate_wakeup_to_parent(dev);
1433
1434Skip:
1435 dev->power.is_late_suspended = true;
1436
1437Complete:
1438 TRACE_SUSPEND(error);
1439 complete_all(&dev->power.completion);
1440 return error;
1441}
1442
1443static void async_suspend_late(void *data, async_cookie_t cookie)
1444{
1445 struct device *dev = data;
1446
1447 device_suspend_late(dev, pm_transition, true);
1448 put_device(dev);
1449}
1450
1451/**
1452 * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1453 * @state: PM transition of the system being carried out.
1454 */
1455int dpm_suspend_late(pm_message_t state)
1456{
1457 ktime_t starttime = ktime_get();
1458 int error = 0;
1459
1460 trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1461
1462 pm_transition = state;
1463 async_error = 0;
1464
1465 wake_up_all_idle_cpus();
1466
1467 mutex_lock(&dpm_list_mtx);
1468
1469 while (!list_empty(&dpm_suspended_list)) {
1470 struct device *dev = to_device(dpm_suspended_list.prev);
1471
1472 list_move(&dev->power.entry, &dpm_late_early_list);
1473
1474 if (dpm_async_fn(dev, async_suspend_late))
1475 continue;
1476
1477 get_device(dev);
1478
1479 mutex_unlock(&dpm_list_mtx);
1480
1481 error = device_suspend_late(dev, state, false);
1482
1483 put_device(dev);
1484
1485 mutex_lock(&dpm_list_mtx);
1486
1487 if (error || async_error)
1488 break;
1489 }
1490
1491 mutex_unlock(&dpm_list_mtx);
1492
1493 async_synchronize_full();
1494 if (!error)
1495 error = async_error;
1496
1497 if (error) {
1498 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1499 dpm_resume_early(resume_event(state));
1500 }
1501 dpm_show_time(starttime, state, error, "late");
1502 trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1503 return error;
1504}
1505
1506/**
1507 * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1508 * @state: PM transition of the system being carried out.
1509 */
1510int dpm_suspend_end(pm_message_t state)
1511{
1512 ktime_t starttime = ktime_get();
1513 int error;
1514
1515 error = dpm_suspend_late(state);
1516 if (error)
1517 goto out;
1518
1519 error = dpm_suspend_noirq(state);
1520 if (error)
1521 dpm_resume_early(resume_event(state));
1522
1523out:
1524 dpm_show_time(starttime, state, error, "end");
1525 return error;
1526}
1527EXPORT_SYMBOL_GPL(dpm_suspend_end);
1528
1529/**
1530 * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1531 * @dev: Device to suspend.
1532 * @state: PM transition of the system being carried out.
1533 * @cb: Suspend callback to execute.
1534 * @info: string description of caller.
1535 */
1536static int legacy_suspend(struct device *dev, pm_message_t state,
1537 int (*cb)(struct device *dev, pm_message_t state),
1538 const char *info)
1539{
1540 int error;
1541 ktime_t calltime;
1542
1543 calltime = initcall_debug_start(dev, cb);
1544
1545 trace_device_pm_callback_start(dev, info, state.event);
1546 error = cb(dev, state);
1547 trace_device_pm_callback_end(dev, error);
1548 suspend_report_result(dev, cb, error);
1549
1550 initcall_debug_report(dev, calltime, cb, error);
1551
1552 return error;
1553}
1554
1555static void dpm_clear_superiors_direct_complete(struct device *dev)
1556{
1557 struct device_link *link;
1558 int idx;
1559
1560 if (dev->parent) {
1561 spin_lock_irq(&dev->parent->power.lock);
1562 dev->parent->power.direct_complete = false;
1563 spin_unlock_irq(&dev->parent->power.lock);
1564 }
1565
1566 idx = device_links_read_lock();
1567
1568 list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
1569 spin_lock_irq(&link->supplier->power.lock);
1570 link->supplier->power.direct_complete = false;
1571 spin_unlock_irq(&link->supplier->power.lock);
1572 }
1573
1574 device_links_read_unlock(idx);
1575}
1576
1577/**
1578 * device_suspend - Execute "suspend" callbacks for given device.
1579 * @dev: Device to handle.
1580 * @state: PM transition of the system being carried out.
1581 * @async: If true, the device is being suspended asynchronously.
1582 */
1583static int device_suspend(struct device *dev, pm_message_t state, bool async)
1584{
1585 pm_callback_t callback = NULL;
1586 const char *info = NULL;
1587 int error = 0;
1588 DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1589
1590 TRACE_DEVICE(dev);
1591 TRACE_SUSPEND(0);
1592
1593 dpm_wait_for_subordinate(dev, async);
1594
1595 if (async_error) {
1596 dev->power.direct_complete = false;
1597 goto Complete;
1598 }
1599
1600 /*
1601 * Wait for possible runtime PM transitions of the device in progress
1602 * to complete and if there's a runtime resume request pending for it,
1603 * resume it before proceeding with invoking the system-wide suspend
1604 * callbacks for it.
1605 *
1606 * If the system-wide suspend callbacks below change the configuration
1607 * of the device, they must disable runtime PM for it or otherwise
1608 * ensure that its runtime-resume callbacks will not be confused by that
1609 * change in case they are invoked going forward.
1610 */
1611 pm_runtime_barrier(dev);
1612
1613 if (pm_wakeup_pending()) {
1614 dev->power.direct_complete = false;
1615 async_error = -EBUSY;
1616 goto Complete;
1617 }
1618
1619 if (dev->power.syscore)
1620 goto Complete;
1621
1622 /* Avoid direct_complete to let wakeup_path propagate. */
1623 if (device_may_wakeup(dev) || device_wakeup_path(dev))
1624 dev->power.direct_complete = false;
1625
1626 if (dev->power.direct_complete) {
1627 if (pm_runtime_status_suspended(dev)) {
1628 pm_runtime_disable(dev);
1629 if (pm_runtime_status_suspended(dev)) {
1630 pm_dev_dbg(dev, state, "direct-complete ");
1631 goto Complete;
1632 }
1633
1634 pm_runtime_enable(dev);
1635 }
1636 dev->power.direct_complete = false;
1637 }
1638
1639 dev->power.may_skip_resume = true;
1640 dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);
1641
1642 dpm_watchdog_set(&wd, dev);
1643 device_lock(dev);
1644
1645 if (dev->pm_domain) {
1646 info = "power domain ";
1647 callback = pm_op(&dev->pm_domain->ops, state);
1648 goto Run;
1649 }
1650
1651 if (dev->type && dev->type->pm) {
1652 info = "type ";
1653 callback = pm_op(dev->type->pm, state);
1654 goto Run;
1655 }
1656
1657 if (dev->class && dev->class->pm) {
1658 info = "class ";
1659 callback = pm_op(dev->class->pm, state);
1660 goto Run;
1661 }
1662
1663 if (dev->bus) {
1664 if (dev->bus->pm) {
1665 info = "bus ";
1666 callback = pm_op(dev->bus->pm, state);
1667 } else if (dev->bus->suspend) {
1668 pm_dev_dbg(dev, state, "legacy bus ");
1669 error = legacy_suspend(dev, state, dev->bus->suspend,
1670 "legacy bus ");
1671 goto End;
1672 }
1673 }
1674
1675 Run:
1676 if (!callback && dev->driver && dev->driver->pm) {
1677 info = "driver ";
1678 callback = pm_op(dev->driver->pm, state);
1679 }
1680
1681 error = dpm_run_callback(callback, dev, state, info);
1682
1683 End:
1684 if (!error) {
1685 dev->power.is_suspended = true;
1686 if (device_may_wakeup(dev))
1687 dev->power.wakeup_path = true;
1688
1689 dpm_propagate_wakeup_to_parent(dev);
1690 dpm_clear_superiors_direct_complete(dev);
1691 }
1692
1693 device_unlock(dev);
1694 dpm_watchdog_clear(&wd);
1695
1696 Complete:
1697 if (error) {
1698 async_error = error;
1699 dpm_save_failed_dev(dev_name(dev));
1700 pm_dev_err(dev, state, async ? " async" : "", error);
1701 }
1702
1703 complete_all(&dev->power.completion);
1704 TRACE_SUSPEND(error);
1705 return error;
1706}
1707
1708static void async_suspend(void *data, async_cookie_t cookie)
1709{
1710 struct device *dev = data;
1711
1712 device_suspend(dev, pm_transition, true);
1713 put_device(dev);
1714}
1715
1716/**
1717 * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1718 * @state: PM transition of the system being carried out.
1719 */
1720int dpm_suspend(pm_message_t state)
1721{
1722 ktime_t starttime = ktime_get();
1723 int error = 0;
1724
1725 trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1726 might_sleep();
1727
1728 devfreq_suspend();
1729 cpufreq_suspend();
1730
1731 pm_transition = state;
1732 async_error = 0;
1733
1734 mutex_lock(&dpm_list_mtx);
1735
1736 while (!list_empty(&dpm_prepared_list)) {
1737 struct device *dev = to_device(dpm_prepared_list.prev);
1738
1739 list_move(&dev->power.entry, &dpm_suspended_list);
1740
1741 if (dpm_async_fn(dev, async_suspend))
1742 continue;
1743
1744 get_device(dev);
1745
1746 mutex_unlock(&dpm_list_mtx);
1747
1748 error = device_suspend(dev, state, false);
1749
1750 put_device(dev);
1751
1752 mutex_lock(&dpm_list_mtx);
1753
1754 if (error || async_error)
1755 break;
1756 }
1757
1758 mutex_unlock(&dpm_list_mtx);
1759
1760 async_synchronize_full();
1761 if (!error)
1762 error = async_error;
1763
1764 if (error)
1765 dpm_save_failed_step(SUSPEND_SUSPEND);
1766
1767 dpm_show_time(starttime, state, error, NULL);
1768 trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1769 return error;
1770}
1771
1772/**
1773 * device_prepare - Prepare a device for system power transition.
1774 * @dev: Device to handle.
1775 * @state: PM transition of the system being carried out.
1776 *
1777 * Execute the ->prepare() callback(s) for given device. No new children of the
1778 * device may be registered after this function has returned.
1779 */
1780static int device_prepare(struct device *dev, pm_message_t state)
1781{
1782 int (*callback)(struct device *) = NULL;
1783 int ret = 0;
1784
1785 /*
1786 * If a device's parent goes into runtime suspend at the wrong time,
1787 * it won't be possible to resume the device. To prevent this we
1788 * block runtime suspend here, during the prepare phase, and allow
1789 * it again during the complete phase.
1790 */
1791 pm_runtime_get_noresume(dev);
1792
1793 if (dev->power.syscore)
1794 return 0;
1795
1796 device_lock(dev);
1797
1798 dev->power.wakeup_path = false;
1799
1800 if (dev->power.no_pm_callbacks)
1801 goto unlock;
1802
1803 if (dev->pm_domain)
1804 callback = dev->pm_domain->ops.prepare;
1805 else if (dev->type && dev->type->pm)
1806 callback = dev->type->pm->prepare;
1807 else if (dev->class && dev->class->pm)
1808 callback = dev->class->pm->prepare;
1809 else if (dev->bus && dev->bus->pm)
1810 callback = dev->bus->pm->prepare;
1811
1812 if (!callback && dev->driver && dev->driver->pm)
1813 callback = dev->driver->pm->prepare;
1814
1815 if (callback)
1816 ret = callback(dev);
1817
1818unlock:
1819 device_unlock(dev);
1820
1821 if (ret < 0) {
1822 suspend_report_result(dev, callback, ret);
1823 pm_runtime_put(dev);
1824 return ret;
1825 }
1826 /*
1827 * A positive return value from ->prepare() means "this device appears
1828 * to be runtime-suspended and its state is fine, so if it really is
1829 * runtime-suspended, you can leave it in that state provided that you
1830 * will do the same thing with all of its descendants". This only
1831 * applies to suspend transitions, however.
1832 */
1833 spin_lock_irq(&dev->power.lock);
1834 dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
1835 (ret > 0 || dev->power.no_pm_callbacks) &&
1836 !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
1837 spin_unlock_irq(&dev->power.lock);
1838 return 0;
1839}
1840
1841/**
1842 * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1843 * @state: PM transition of the system being carried out.
1844 *
1845 * Execute the ->prepare() callback(s) for all devices.
1846 */
1847int dpm_prepare(pm_message_t state)
1848{
1849 int error = 0;
1850
1851 trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1852 might_sleep();
1853
1854 /*
1855 * Give a chance for the known devices to complete their probes, before
1856 * disable probing of devices. This sync point is important at least
1857 * at boot time + hibernation restore.
1858 */
1859 wait_for_device_probe();
1860 /*
1861 * It is unsafe if probing of devices will happen during suspend or
1862 * hibernation and system behavior will be unpredictable in this case.
1863 * So, let's prohibit device's probing here and defer their probes
1864 * instead. The normal behavior will be restored in dpm_complete().
1865 */
1866 device_block_probing();
1867
1868 mutex_lock(&dpm_list_mtx);
1869 while (!list_empty(&dpm_list) && !error) {
1870 struct device *dev = to_device(dpm_list.next);
1871
1872 get_device(dev);
1873
1874 mutex_unlock(&dpm_list_mtx);
1875
1876 trace_device_pm_callback_start(dev, "", state.event);
1877 error = device_prepare(dev, state);
1878 trace_device_pm_callback_end(dev, error);
1879
1880 mutex_lock(&dpm_list_mtx);
1881
1882 if (!error) {
1883 dev->power.is_prepared = true;
1884 if (!list_empty(&dev->power.entry))
1885 list_move_tail(&dev->power.entry, &dpm_prepared_list);
1886 } else if (error == -EAGAIN) {
1887 error = 0;
1888 } else {
1889 dev_info(dev, "not prepared for power transition: code %d\n",
1890 error);
1891 }
1892
1893 mutex_unlock(&dpm_list_mtx);
1894
1895 put_device(dev);
1896
1897 mutex_lock(&dpm_list_mtx);
1898 }
1899 mutex_unlock(&dpm_list_mtx);
1900 trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
1901 return error;
1902}
1903
1904/**
1905 * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1906 * @state: PM transition of the system being carried out.
1907 *
1908 * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1909 * callbacks for them.
1910 */
1911int dpm_suspend_start(pm_message_t state)
1912{
1913 ktime_t starttime = ktime_get();
1914 int error;
1915
1916 error = dpm_prepare(state);
1917 if (error)
1918 dpm_save_failed_step(SUSPEND_PREPARE);
1919 else
1920 error = dpm_suspend(state);
1921
1922 dpm_show_time(starttime, state, error, "start");
1923 return error;
1924}
1925EXPORT_SYMBOL_GPL(dpm_suspend_start);
1926
1927void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret)
1928{
1929 if (ret)
1930 dev_err(dev, "%s(): %pS returns %d\n", function, fn, ret);
1931}
1932EXPORT_SYMBOL_GPL(__suspend_report_result);
1933
1934/**
1935 * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1936 * @subordinate: Device that needs to wait for @dev.
1937 * @dev: Device to wait for.
1938 */
1939int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1940{
1941 dpm_wait(dev, subordinate->power.async_suspend);
1942 return async_error;
1943}
1944EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1945
1946/**
1947 * dpm_for_each_dev - device iterator.
1948 * @data: data for the callback.
1949 * @fn: function to be called for each device.
1950 *
1951 * Iterate over devices in dpm_list, and call @fn for each device,
1952 * passing it @data.
1953 */
1954void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1955{
1956 struct device *dev;
1957
1958 if (!fn)
1959 return;
1960
1961 device_pm_lock();
1962 list_for_each_entry(dev, &dpm_list, power.entry)
1963 fn(dev, data);
1964 device_pm_unlock();
1965}
1966EXPORT_SYMBOL_GPL(dpm_for_each_dev);
1967
1968static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
1969{
1970 if (!ops)
1971 return true;
1972
1973 return !ops->prepare &&
1974 !ops->suspend &&
1975 !ops->suspend_late &&
1976 !ops->suspend_noirq &&
1977 !ops->resume_noirq &&
1978 !ops->resume_early &&
1979 !ops->resume &&
1980 !ops->complete;
1981}
1982
1983void device_pm_check_callbacks(struct device *dev)
1984{
1985 unsigned long flags;
1986
1987 spin_lock_irqsave(&dev->power.lock, flags);
1988 dev->power.no_pm_callbacks =
1989 (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
1990 !dev->bus->suspend && !dev->bus->resume)) &&
1991 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
1992 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
1993 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
1994 (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
1995 !dev->driver->suspend && !dev->driver->resume));
1996 spin_unlock_irqrestore(&dev->power.lock, flags);
1997}
1998
1999bool dev_pm_skip_suspend(struct device *dev)
2000{
2001 return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
2002 pm_runtime_status_suspended(dev);
2003}