1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * drivers/base/power/runtime.c - Helper functions for device runtime PM
   4 *
   5 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
   6 * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
   7 */
   8#include <linux/sched/mm.h>
   9#include <linux/ktime.h>
  10#include <linux/hrtimer.h>
  11#include <linux/export.h>
  12#include <linux/pm_runtime.h>
  13#include <linux/pm_wakeirq.h>
  14#include <linux/rculist.h>
  15#include <trace/events/rpm.h>
  16
  17#include "../base.h"
  18#include "power.h"
  19
  20typedef int (*pm_callback_t)(struct device *);
  21
  22static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
  23{
  24	pm_callback_t cb;
  25	const struct dev_pm_ops *ops;
  26
  27	if (dev->pm_domain)
  28		ops = &dev->pm_domain->ops;
  29	else if (dev->type && dev->type->pm)
  30		ops = dev->type->pm;
  31	else if (dev->class && dev->class->pm)
  32		ops = dev->class->pm;
  33	else if (dev->bus && dev->bus->pm)
  34		ops = dev->bus->pm;
  35	else
  36		ops = NULL;
  37
  38	if (ops)
  39		cb = *(pm_callback_t *)((void *)ops + cb_offset);
  40	else
  41		cb = NULL;
  42
  43	if (!cb && dev->driver && dev->driver->pm)
  44		cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
  45
  46	return cb;
  47}
  48
  49#define RPM_GET_CALLBACK(dev, callback) \
  50		__rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
  51
  52static int rpm_resume(struct device *dev, int rpmflags);
  53static int rpm_suspend(struct device *dev, int rpmflags);
  54
  55/**
  56 * update_pm_runtime_accounting - Update the time accounting of power states
  57 * @dev: Device to update the accounting for
  58 *
  59 * In order to be able to have time accounting of the various power states
  60 * (as used by programs such as PowerTOP to show the effectiveness of runtime
  61 * PM), we need to track the time spent in each state.
  62 * update_pm_runtime_accounting must be called each time before the
  63 * runtime_status field is updated, to account the time in the old state
  64 * correctly.
  65 */
  66static void update_pm_runtime_accounting(struct device *dev)
  67{
  68	u64 now, last, delta;
  69
  70	if (dev->power.disable_depth > 0)
  71		return;
  72
  73	last = dev->power.accounting_timestamp;
  74
  75	now = ktime_get_mono_fast_ns();
  76	dev->power.accounting_timestamp = now;
  77
  78	/*
  79	 * Because ktime_get_mono_fast_ns() is not monotonic during
  80	 * timekeeping updates, ensure that 'now' is after the last saved
  81	 * timesptamp.
  82	 */
  83	if (now < last)
  84		return;
  85
  86	delta = now - last;
  87
  88	if (dev->power.runtime_status == RPM_SUSPENDED)
  89		dev->power.suspended_time += delta;
  90	else
  91		dev->power.active_time += delta;
  92}
  93
  94static void __update_runtime_status(struct device *dev, enum rpm_status status)
  95{
  96	update_pm_runtime_accounting(dev);
  97	trace_rpm_status(dev, status);
  98	dev->power.runtime_status = status;
  99}
 100
 101static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
 102{
 103	u64 time;
 104	unsigned long flags;
 105
 106	spin_lock_irqsave(&dev->power.lock, flags);
 107
 108	update_pm_runtime_accounting(dev);
 109	time = suspended ? dev->power.suspended_time : dev->power.active_time;
 110
 111	spin_unlock_irqrestore(&dev->power.lock, flags);
 112
 113	return time;
 114}
 115
 116u64 pm_runtime_active_time(struct device *dev)
 117{
 118	return rpm_get_accounted_time(dev, false);
 119}
 120
 121u64 pm_runtime_suspended_time(struct device *dev)
 122{
 123	return rpm_get_accounted_time(dev, true);
 124}
 125EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
 126
 127/**
 128 * pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
 129 * @dev: Device to handle.
 130 */
 131static void pm_runtime_deactivate_timer(struct device *dev)
 132{
 133	if (dev->power.timer_expires > 0) {
 134		hrtimer_try_to_cancel(&dev->power.suspend_timer);
 135		dev->power.timer_expires = 0;
 136	}
 137}
 138
 139/**
 140 * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
 141 * @dev: Device to handle.
 142 */
 143static void pm_runtime_cancel_pending(struct device *dev)
 144{
 145	pm_runtime_deactivate_timer(dev);
 146	/*
 147	 * In case there's a request pending, make sure its work function will
 148	 * return without doing anything.
 149	 */
 150	dev->power.request = RPM_REQ_NONE;
 151}
 152
 153/*
 154 * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
 155 * @dev: Device to handle.
 156 *
 157 * Compute the autosuspend-delay expiration time based on the device's
 158 * power.last_busy time.  If the delay has already expired or is disabled
 159 * (negative) or the power.use_autosuspend flag isn't set, return 0.
 160 * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
 161 *
 162 * This function may be called either with or without dev->power.lock held.
 163 * Either way it can be racy, since power.last_busy may be updated at any time.
 164 */
 165u64 pm_runtime_autosuspend_expiration(struct device *dev)
 166{
 167	int autosuspend_delay;
 168	u64 expires;
 169
 170	if (!dev->power.use_autosuspend)
 171		return 0;
 172
 173	autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
 174	if (autosuspend_delay < 0)
 175		return 0;
 176
 177	expires  = READ_ONCE(dev->power.last_busy);
 178	expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
 179	if (expires > ktime_get_mono_fast_ns())
 180		return expires;	/* Expires in the future */
 181
 182	return 0;
 183}
 184EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
 185
 186static int dev_memalloc_noio(struct device *dev, void *data)
 187{
 188	return dev->power.memalloc_noio;
 189}
 190
 191/*
 192 * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
 193 * @dev: Device to handle.
 194 * @enable: True for setting the flag and False for clearing the flag.
 195 *
 196 * Set the flag for all devices in the path from the device to the
 197 * root device in the device tree if @enable is true, otherwise clear
 198 * the flag for devices in the path whose siblings don't set the flag.
 199 *
 200 * The function should only be called by block device, or network
 201 * device driver for solving the deadlock problem during runtime
 202 * resume/suspend:
 203 *
 204 *     If memory allocation with GFP_KERNEL is called inside runtime
 205 *     resume/suspend callback of any one of its ancestors(or the
 206 *     block device itself), the deadlock may be triggered inside the
 207 *     memory allocation since it might not complete until the block
 208 *     device becomes active and the involed page I/O finishes. The
 209 *     situation is pointed out first by Alan Stern. Network device
 210 *     are involved in iSCSI kind of situation.
 211 *
 212 * The lock of dev_hotplug_mutex is held in the function for handling
 213 * hotplug race because pm_runtime_set_memalloc_noio() may be called
 214 * in async probe().
 215 *
 216 * The function should be called between device_add() and device_del()
 217 * on the affected device(block/network device).
 218 */
 219void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
 220{
 221	static DEFINE_MUTEX(dev_hotplug_mutex);
 222
 223	mutex_lock(&dev_hotplug_mutex);
 224	for (;;) {
 225		bool enabled;
 226
 227		/* hold power lock since bitfield is not SMP-safe. */
 228		spin_lock_irq(&dev->power.lock);
 229		enabled = dev->power.memalloc_noio;
 230		dev->power.memalloc_noio = enable;
 231		spin_unlock_irq(&dev->power.lock);
 232
 233		/*
 234		 * not need to enable ancestors any more if the device
 235		 * has been enabled.
 236		 */
 237		if (enabled && enable)
 238			break;
 239
 240		dev = dev->parent;
 241
 242		/*
 243		 * clear flag of the parent device only if all the
 244		 * children don't set the flag because ancestor's
 245		 * flag was set by any one of the descendants.
 246		 */
 247		if (!dev || (!enable &&
 248		    device_for_each_child(dev, NULL, dev_memalloc_noio)))
 
 249			break;
 250	}
 251	mutex_unlock(&dev_hotplug_mutex);
 252}
 253EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
 254
 255/**
 256 * rpm_check_suspend_allowed - Test whether a device may be suspended.
 257 * @dev: Device to test.
 258 */
 259static int rpm_check_suspend_allowed(struct device *dev)
 260{
 261	int retval = 0;
 262
 263	if (dev->power.runtime_error)
 264		retval = -EINVAL;
 265	else if (dev->power.disable_depth > 0)
 266		retval = -EACCES;
 267	else if (atomic_read(&dev->power.usage_count))
 268		retval = -EAGAIN;
 269	else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count))
 
 270		retval = -EBUSY;
 271
 272	/* Pending resume requests take precedence over suspends. */
 273	else if ((dev->power.deferred_resume &&
 274	    dev->power.runtime_status == RPM_SUSPENDING) ||
 275	    (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME))
 
 276		retval = -EAGAIN;
 277	else if (__dev_pm_qos_resume_latency(dev) == 0)
 278		retval = -EPERM;
 279	else if (dev->power.runtime_status == RPM_SUSPENDED)
 280		retval = 1;
 281
 282	return retval;
 283}
 284
 285static int rpm_get_suppliers(struct device *dev)
 286{
 287	struct device_link *link;
 288
 289	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 290				device_links_read_lock_held()) {
 291		int retval;
 292
 293		if (!(link->flags & DL_FLAG_PM_RUNTIME))
 
 294			continue;
 295
 296		retval = pm_runtime_get_sync(link->supplier);
 297		/* Ignore suppliers with disabled runtime PM. */
 298		if (retval < 0 && retval != -EACCES) {
 299			pm_runtime_put_noidle(link->supplier);
 300			return retval;
 301		}
 302		refcount_inc(&link->rpm_active);
 303	}
 304	return 0;
 305}
 306
 307/**
 308 * pm_runtime_release_supplier - Drop references to device link's supplier.
 309 * @link: Target device link.
 310 *
 311 * Drop all runtime PM references associated with @link to its supplier device.
 312 */
 313void pm_runtime_release_supplier(struct device_link *link)
 314{
 315	struct device *supplier = link->supplier;
 316
 317	/*
 318	 * The additional power.usage_count check is a safety net in case
 319	 * the rpm_active refcount becomes saturated, in which case
 320	 * refcount_dec_not_one() would return true forever, but it is not
 321	 * strictly necessary.
 322	 */
 323	while (refcount_dec_not_one(&link->rpm_active) &&
 324	       atomic_read(&supplier->power.usage_count) > 0)
 325		pm_runtime_put_noidle(supplier);
 326}
 327
 328static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
 329{
 330	struct device_link *link;
 331
 332	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 333				device_links_read_lock_held()) {
 334		pm_runtime_release_supplier(link);
 335		if (try_to_suspend)
 336			pm_request_idle(link->supplier);
 337	}
 338}
 339
 340static void rpm_put_suppliers(struct device *dev)
 341{
 342	__rpm_put_suppliers(dev, true);
 343}
 344
 345static void rpm_suspend_suppliers(struct device *dev)
 346{
 347	struct device_link *link;
 348	int idx = device_links_read_lock();
 349
 350	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 351				device_links_read_lock_held())
 352		pm_request_idle(link->supplier);
 
 353
 354	device_links_read_unlock(idx);
 
 
 355}
 356
 357/**
 358 * __rpm_callback - Run a given runtime PM callback for a given device.
 359 * @cb: Runtime PM callback to run.
 360 * @dev: Device to run the callback for.
 361 */
 362static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
 363	__releases(&dev->power.lock) __acquires(&dev->power.lock)
 364{
 365	int retval = 0, idx;
 366	bool use_links = dev->power.links_count > 0;
 367
 368	if (dev->power.irq_safe) {
 369		spin_unlock(&dev->power.lock);
 370	} else {
 371		spin_unlock_irq(&dev->power.lock);
 372
 373		/*
 374		 * Resume suppliers if necessary.
 375		 *
 376		 * The device's runtime PM status cannot change until this
 377		 * routine returns, so it is safe to read the status outside of
 378		 * the lock.
 379		 */
 380		if (use_links && dev->power.runtime_status == RPM_RESUMING) {
 381			idx = device_links_read_lock();
 382
 383			retval = rpm_get_suppliers(dev);
 384			if (retval) {
 385				rpm_put_suppliers(dev);
 386				goto fail;
 387			}
 388
 389			device_links_read_unlock(idx);
 390		}
 391	}
 392
 393	if (cb)
 394		retval = cb(dev);
 395
 396	if (dev->power.irq_safe) {
 397		spin_lock(&dev->power.lock);
 398	} else {
 399		/*
 400		 * If the device is suspending and the callback has returned
 401		 * success, drop the usage counters of the suppliers that have
 402		 * been reference counted on its resume.
 403		 *
 404		 * Do that if resume fails too.
 405		 */
 406		if (use_links &&
 407		    ((dev->power.runtime_status == RPM_SUSPENDING && !retval) ||
 408		    (dev->power.runtime_status == RPM_RESUMING && retval))) {
 409			idx = device_links_read_lock();
 410
 411			__rpm_put_suppliers(dev, false);
 
 412
 413fail:
 414			device_links_read_unlock(idx);
 415		}
 416
 417		spin_lock_irq(&dev->power.lock);
 418	}
 419
 420	return retval;
 421}
 422
 423/**
 424 * rpm_callback - Run a given runtime PM callback for a given device.
 425 * @cb: Runtime PM callback to run.
 426 * @dev: Device to run the callback for.
 427 */
 428static int rpm_callback(int (*cb)(struct device *), struct device *dev)
 429{
 430	int retval;
 431
 432	if (dev->power.memalloc_noio) {
 433		unsigned int noio_flag;
 434
 435		/*
 436		 * Deadlock might be caused if memory allocation with
 437		 * GFP_KERNEL happens inside runtime_suspend and
 438		 * runtime_resume callbacks of one block device's
 439		 * ancestor or the block device itself. Network
 440		 * device might be thought as part of iSCSI block
 441		 * device, so network device and its ancestor should
 442		 * be marked as memalloc_noio too.
 443		 */
 444		noio_flag = memalloc_noio_save();
 445		retval = __rpm_callback(cb, dev);
 446		memalloc_noio_restore(noio_flag);
 447	} else {
 448		retval = __rpm_callback(cb, dev);
 449	}
 450
 451	dev->power.runtime_error = retval;
 452	return retval != -EACCES ? retval : -EIO;
 453}
 454
 455/**
 456 * rpm_idle - Notify device bus type if the device can be suspended.
 457 * @dev: Device to notify the bus type about.
 458 * @rpmflags: Flag bits.
 459 *
 460 * Check if the device's runtime PM status allows it to be suspended.  If
 461 * another idle notification has been started earlier, return immediately.  If
 462 * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
 463 * run the ->runtime_idle() callback directly. If the ->runtime_idle callback
 464 * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
 465 *
 466 * This function must be called under dev->power.lock with interrupts disabled.
 467 */
 468static int rpm_idle(struct device *dev, int rpmflags)
 469{
 470	int (*callback)(struct device *);
 471	int retval;
 472
 473	trace_rpm_idle(dev, rpmflags);
 474	retval = rpm_check_suspend_allowed(dev);
 475	if (retval < 0)
 476		;	/* Conditions are wrong. */
 477
 478	/* Idle notifications are allowed only in the RPM_ACTIVE state. */
 479	else if (dev->power.runtime_status != RPM_ACTIVE)
 480		retval = -EAGAIN;
 481
 482	/*
 483	 * Any pending request other than an idle notification takes
 484	 * precedence over us, except that the timer may be running.
 485	 */
 486	else if (dev->power.request_pending &&
 487	    dev->power.request > RPM_REQ_IDLE)
 488		retval = -EAGAIN;
 489
 490	/* Act as though RPM_NOWAIT is always set. */
 491	else if (dev->power.idle_notification)
 492		retval = -EINPROGRESS;
 493
 494	if (retval)
 495		goto out;
 496
 497	/* Pending requests need to be canceled. */
 498	dev->power.request = RPM_REQ_NONE;
 499
 500	callback = RPM_GET_CALLBACK(dev, runtime_idle);
 501
 502	/* If no callback assume success. */
 503	if (!callback || dev->power.no_callbacks)
 504		goto out;
 505
 506	/* Carry out an asynchronous or a synchronous idle notification. */
 507	if (rpmflags & RPM_ASYNC) {
 508		dev->power.request = RPM_REQ_IDLE;
 509		if (!dev->power.request_pending) {
 510			dev->power.request_pending = true;
 511			queue_work(pm_wq, &dev->power.work);
 512		}
 513		trace_rpm_return_int(dev, _THIS_IP_, 0);
 514		return 0;
 515	}
 516
 517	dev->power.idle_notification = true;
 518
 519	if (dev->power.irq_safe)
 520		spin_unlock(&dev->power.lock);
 521	else
 522		spin_unlock_irq(&dev->power.lock);
 523
 524	retval = callback(dev);
 525
 526	if (dev->power.irq_safe)
 527		spin_lock(&dev->power.lock);
 528	else
 529		spin_lock_irq(&dev->power.lock);
 530
 531	dev->power.idle_notification = false;
 532	wake_up_all(&dev->power.wait_queue);
 533
 534 out:
 535	trace_rpm_return_int(dev, _THIS_IP_, retval);
 536	return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
 537}
 538
 539/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 540 * rpm_suspend - Carry out runtime suspend of given device.
 541 * @dev: Device to suspend.
 542 * @rpmflags: Flag bits.
 543 *
 544 * Check if the device's runtime PM status allows it to be suspended.
 545 * Cancel a pending idle notification, autosuspend or suspend. If
 546 * another suspend has been started earlier, either return immediately
 547 * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
 548 * flags. If the RPM_ASYNC flag is set then queue a suspend request;
 549 * otherwise run the ->runtime_suspend() callback directly. When
 550 * ->runtime_suspend succeeded, if a deferred resume was requested while
 551 * the callback was running then carry it out, otherwise send an idle
 552 * notification for its parent (if the suspend succeeded and both
 553 * ignore_children of parent->power and irq_safe of dev->power are not set).
 554 * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
 555 * flag is set and the next autosuspend-delay expiration time is in the
 556 * future, schedule another autosuspend attempt.
 557 *
 558 * This function must be called under dev->power.lock with interrupts disabled.
 559 */
 560static int rpm_suspend(struct device *dev, int rpmflags)
 561	__releases(&dev->power.lock) __acquires(&dev->power.lock)
 562{
 563	int (*callback)(struct device *);
 564	struct device *parent = NULL;
 565	int retval;
 566
 567	trace_rpm_suspend(dev, rpmflags);
 568
 569 repeat:
 570	retval = rpm_check_suspend_allowed(dev);
 
 571	if (retval < 0)
 572		goto out;	/* Conditions are wrong. */
 573
 574	/* Synchronous suspends are not allowed in the RPM_RESUMING state. */
 575	if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
 
 576		retval = -EAGAIN;
 577
 578	if (retval)
 579		goto out;
 580
 581	/* If the autosuspend_delay time hasn't expired yet, reschedule. */
 582	if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) {
 
 583		u64 expires = pm_runtime_autosuspend_expiration(dev);
 584
 585		if (expires != 0) {
 586			/* Pending requests need to be canceled. */
 587			dev->power.request = RPM_REQ_NONE;
 588
 589			/*
 590			 * Optimization: If the timer is already running and is
 591			 * set to expire at or before the autosuspend delay,
 592			 * avoid the overhead of resetting it.  Just let it
 593			 * expire; pm_suspend_timer_fn() will take care of the
 594			 * rest.
 595			 */
 596			if (!(dev->power.timer_expires &&
 597			    dev->power.timer_expires <= expires)) {
 598				/*
 599				 * We add a slack of 25% to gather wakeups
 600				 * without sacrificing the granularity.
 601				 */
 602				u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
 603						    (NSEC_PER_MSEC >> 2);
 604
 605				dev->power.timer_expires = expires;
 606				hrtimer_start_range_ns(&dev->power.suspend_timer,
 607						       ns_to_ktime(expires),
 608						       slack,
 609						       HRTIMER_MODE_ABS);
 610			}
 611			dev->power.timer_autosuspends = 1;
 612			goto out;
 613		}
 614	}
 615
 616	/* Other scheduled or pending requests need to be canceled. */
 617	pm_runtime_cancel_pending(dev);
 618
 619	if (dev->power.runtime_status == RPM_SUSPENDING) {
 620		DEFINE_WAIT(wait);
 621
 622		if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
 623			retval = -EINPROGRESS;
 624			goto out;
 625		}
 626
 627		if (dev->power.irq_safe) {
 628			spin_unlock(&dev->power.lock);
 629
 630			cpu_relax();
 631
 632			spin_lock(&dev->power.lock);
 633			goto repeat;
 634		}
 635
 636		/* Wait for the other suspend running in parallel with us. */
 637		for (;;) {
 638			prepare_to_wait(&dev->power.wait_queue, &wait,
 639					TASK_UNINTERRUPTIBLE);
 640			if (dev->power.runtime_status != RPM_SUSPENDING)
 641				break;
 642
 643			spin_unlock_irq(&dev->power.lock);
 644
 645			schedule();
 646
 647			spin_lock_irq(&dev->power.lock);
 648		}
 649		finish_wait(&dev->power.wait_queue, &wait);
 650		goto repeat;
 651	}
 652
 653	if (dev->power.no_callbacks)
 654		goto no_callback;	/* Assume success. */
 655
 656	/* Carry out an asynchronous or a synchronous suspend. */
 657	if (rpmflags & RPM_ASYNC) {
 658		dev->power.request = (rpmflags & RPM_AUTO) ?
 659		    RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
 660		if (!dev->power.request_pending) {
 661			dev->power.request_pending = true;
 662			queue_work(pm_wq, &dev->power.work);
 663		}
 664		goto out;
 665	}
 666
 667	__update_runtime_status(dev, RPM_SUSPENDING);
 668
 669	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
 670
 671	dev_pm_enable_wake_irq_check(dev, true);
 672	retval = rpm_callback(callback, dev);
 673	if (retval)
 674		goto fail;
 675
 676	dev_pm_enable_wake_irq_complete(dev);
 677
 678 no_callback:
 679	__update_runtime_status(dev, RPM_SUSPENDED);
 680	pm_runtime_deactivate_timer(dev);
 681
 682	if (dev->parent) {
 683		parent = dev->parent;
 684		atomic_add_unless(&parent->power.child_count, -1, 0);
 685	}
 686	wake_up_all(&dev->power.wait_queue);
 687
 688	if (dev->power.deferred_resume) {
 689		dev->power.deferred_resume = false;
 690		rpm_resume(dev, 0);
 691		retval = -EAGAIN;
 692		goto out;
 693	}
 694
 695	if (dev->power.irq_safe)
 696		goto out;
 697
 698	/* Maybe the parent is now able to suspend. */
 699	if (parent && !parent->power.ignore_children) {
 700		spin_unlock(&dev->power.lock);
 701
 702		spin_lock(&parent->power.lock);
 703		rpm_idle(parent, RPM_ASYNC);
 704		spin_unlock(&parent->power.lock);
 705
 706		spin_lock(&dev->power.lock);
 707	}
 708	/* Maybe the suppliers are now able to suspend. */
 709	if (dev->power.links_count > 0) {
 710		spin_unlock_irq(&dev->power.lock);
 711
 712		rpm_suspend_suppliers(dev);
 713
 714		spin_lock_irq(&dev->power.lock);
 715	}
 716
 717 out:
 718	trace_rpm_return_int(dev, _THIS_IP_, retval);
 719
 720	return retval;
 721
 722 fail:
 723	dev_pm_disable_wake_irq_check(dev, true);
 724	__update_runtime_status(dev, RPM_ACTIVE);
 725	dev->power.deferred_resume = false;
 726	wake_up_all(&dev->power.wait_queue);
 727
 728	if (retval == -EAGAIN || retval == -EBUSY) {
 729		dev->power.runtime_error = 0;
 730
 731		/*
 732		 * If the callback routine failed an autosuspend, and
 733		 * if the last_busy time has been updated so that there
 734		 * is a new autosuspend expiration time, automatically
 735		 * reschedule another autosuspend.
 736		 */
 737		if ((rpmflags & RPM_AUTO) &&
 738		    pm_runtime_autosuspend_expiration(dev) != 0)
 739			goto repeat;
 740	} else {
 741		pm_runtime_cancel_pending(dev);
 742	}
 743	goto out;
 744}
 745
 746/**
 747 * rpm_resume - Carry out runtime resume of given device.
 748 * @dev: Device to resume.
 749 * @rpmflags: Flag bits.
 750 *
 751 * Check if the device's runtime PM status allows it to be resumed.  Cancel
 752 * any scheduled or pending requests.  If another resume has been started
 753 * earlier, either return immediately or wait for it to finish, depending on the
 754 * RPM_NOWAIT and RPM_ASYNC flags.  Similarly, if there's a suspend running in
 755 * parallel with this function, either tell the other process to resume after
 756 * suspending (deferred_resume) or wait for it to finish.  If the RPM_ASYNC
 757 * flag is set then queue a resume request; otherwise run the
 758 * ->runtime_resume() callback directly.  Queue an idle notification for the
 759 * device if the resume succeeded.
 760 *
 761 * This function must be called under dev->power.lock with interrupts disabled.
 762 */
 763static int rpm_resume(struct device *dev, int rpmflags)
 764	__releases(&dev->power.lock) __acquires(&dev->power.lock)
 765{
 766	int (*callback)(struct device *);
 767	struct device *parent = NULL;
 768	int retval = 0;
 769
 770	trace_rpm_resume(dev, rpmflags);
 771
 772 repeat:
 773	if (dev->power.runtime_error) {
 774		retval = -EINVAL;
 775	} else if (dev->power.disable_depth > 0) {
 776		if (dev->power.runtime_status == RPM_ACTIVE &&
 777		    dev->power.last_status == RPM_ACTIVE)
 778			retval = 1;
 779		else
 780			retval = -EACCES;
 781	}
 782	if (retval)
 783		goto out;
 784
 785	/*
 786	 * Other scheduled or pending requests need to be canceled.  Small
 787	 * optimization: If an autosuspend timer is running, leave it running
 788	 * rather than cancelling it now only to restart it again in the near
 789	 * future.
 790	 */
 791	dev->power.request = RPM_REQ_NONE;
 792	if (!dev->power.timer_autosuspends)
 793		pm_runtime_deactivate_timer(dev);
 794
 795	if (dev->power.runtime_status == RPM_ACTIVE) {
 796		retval = 1;
 797		goto out;
 798	}
 799
 800	if (dev->power.runtime_status == RPM_RESUMING ||
 801	    dev->power.runtime_status == RPM_SUSPENDING) {
 802		DEFINE_WAIT(wait);
 803
 804		if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
 805			if (dev->power.runtime_status == RPM_SUSPENDING) {
 806				dev->power.deferred_resume = true;
 807				if (rpmflags & RPM_NOWAIT)
 808					retval = -EINPROGRESS;
 809			} else {
 810				retval = -EINPROGRESS;
 811			}
 812			goto out;
 813		}
 814
 815		if (dev->power.irq_safe) {
 816			spin_unlock(&dev->power.lock);
 817
 818			cpu_relax();
 819
 820			spin_lock(&dev->power.lock);
 821			goto repeat;
 822		}
 823
 824		/* Wait for the operation carried out in parallel with us. */
 825		for (;;) {
 826			prepare_to_wait(&dev->power.wait_queue, &wait,
 827					TASK_UNINTERRUPTIBLE);
 828			if (dev->power.runtime_status != RPM_RESUMING &&
 829			    dev->power.runtime_status != RPM_SUSPENDING)
 830				break;
 831
 832			spin_unlock_irq(&dev->power.lock);
 833
 834			schedule();
 835
 836			spin_lock_irq(&dev->power.lock);
 837		}
 838		finish_wait(&dev->power.wait_queue, &wait);
 839		goto repeat;
 840	}
 841
 842	/*
 843	 * See if we can skip waking up the parent.  This is safe only if
 844	 * power.no_callbacks is set, because otherwise we don't know whether
 845	 * the resume will actually succeed.
 846	 */
 847	if (dev->power.no_callbacks && !parent && dev->parent) {
 848		spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
 849		if (dev->parent->power.disable_depth > 0 ||
 850		    dev->parent->power.ignore_children ||
 851		    dev->parent->power.runtime_status == RPM_ACTIVE) {
 852			atomic_inc(&dev->parent->power.child_count);
 853			spin_unlock(&dev->parent->power.lock);
 854			retval = 1;
 855			goto no_callback;	/* Assume success. */
 856		}
 857		spin_unlock(&dev->parent->power.lock);
 858	}
 859
 860	/* Carry out an asynchronous or a synchronous resume. */
 861	if (rpmflags & RPM_ASYNC) {
 862		dev->power.request = RPM_REQ_RESUME;
 863		if (!dev->power.request_pending) {
 864			dev->power.request_pending = true;
 865			queue_work(pm_wq, &dev->power.work);
 866		}
 867		retval = 0;
 868		goto out;
 869	}
 870
 871	if (!parent && dev->parent) {
 872		/*
 873		 * Increment the parent's usage counter and resume it if
 874		 * necessary.  Not needed if dev is irq-safe; then the
 875		 * parent is permanently resumed.
 876		 */
 877		parent = dev->parent;
 878		if (dev->power.irq_safe)
 879			goto skip_parent;
 880
 881		spin_unlock(&dev->power.lock);
 882
 883		pm_runtime_get_noresume(parent);
 884
 885		spin_lock(&parent->power.lock);
 886		/*
 887		 * Resume the parent if it has runtime PM enabled and not been
 888		 * set to ignore its children.
 889		 */
 890		if (!parent->power.disable_depth &&
 891		    !parent->power.ignore_children) {
 892			rpm_resume(parent, 0);
 893			if (parent->power.runtime_status != RPM_ACTIVE)
 894				retval = -EBUSY;
 895		}
 896		spin_unlock(&parent->power.lock);
 897
 898		spin_lock(&dev->power.lock);
 899		if (retval)
 900			goto out;
 901
 902		goto repeat;
 903	}
 904 skip_parent:
 905
 906	if (dev->power.no_callbacks)
 907		goto no_callback;	/* Assume success. */
 908
 909	__update_runtime_status(dev, RPM_RESUMING);
 910
 911	callback = RPM_GET_CALLBACK(dev, runtime_resume);
 912
 913	dev_pm_disable_wake_irq_check(dev, false);
 914	retval = rpm_callback(callback, dev);
 915	if (retval) {
 916		__update_runtime_status(dev, RPM_SUSPENDED);
 917		pm_runtime_cancel_pending(dev);
 918		dev_pm_enable_wake_irq_check(dev, false);
 919	} else {
 920 no_callback:
 921		__update_runtime_status(dev, RPM_ACTIVE);
 922		pm_runtime_mark_last_busy(dev);
 923		if (parent)
 924			atomic_inc(&parent->power.child_count);
 925	}
 926	wake_up_all(&dev->power.wait_queue);
 927
 928	if (retval >= 0)
 929		rpm_idle(dev, RPM_ASYNC);
 930
 931 out:
 932	if (parent && !dev->power.irq_safe) {
 933		spin_unlock_irq(&dev->power.lock);
 934
 935		pm_runtime_put(parent);
 936
 937		spin_lock_irq(&dev->power.lock);
 938	}
 939
 940	trace_rpm_return_int(dev, _THIS_IP_, retval);
 941
 942	return retval;
 943}
 944
 945/**
 946 * pm_runtime_work - Universal runtime PM work function.
 947 * @work: Work structure used for scheduling the execution of this function.
 948 *
 949 * Use @work to get the device object the work is to be done for, determine what
 950 * is to be done and execute the appropriate runtime PM function.
 951 */
 952static void pm_runtime_work(struct work_struct *work)
 953{
 954	struct device *dev = container_of(work, struct device, power.work);
 955	enum rpm_request req;
 956
 957	spin_lock_irq(&dev->power.lock);
 958
 959	if (!dev->power.request_pending)
 960		goto out;
 961
 962	req = dev->power.request;
 963	dev->power.request = RPM_REQ_NONE;
 964	dev->power.request_pending = false;
 965
 966	switch (req) {
 967	case RPM_REQ_NONE:
 968		break;
 969	case RPM_REQ_IDLE:
 970		rpm_idle(dev, RPM_NOWAIT);
 971		break;
 972	case RPM_REQ_SUSPEND:
 973		rpm_suspend(dev, RPM_NOWAIT);
 974		break;
 975	case RPM_REQ_AUTOSUSPEND:
 976		rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
 977		break;
 978	case RPM_REQ_RESUME:
 979		rpm_resume(dev, RPM_NOWAIT);
 980		break;
 981	}
 982
 983 out:
 984	spin_unlock_irq(&dev->power.lock);
 985}
 986
 987/**
 988 * pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
 989 * @timer: hrtimer used by pm_schedule_suspend().
 990 *
 991 * Check if the time is right and queue a suspend request.
 992 */
 993static enum hrtimer_restart  pm_suspend_timer_fn(struct hrtimer *timer)
 994{
 995	struct device *dev = container_of(timer, struct device, power.suspend_timer);
 996	unsigned long flags;
 997	u64 expires;
 998
 999	spin_lock_irqsave(&dev->power.lock, flags);
1000
1001	expires = dev->power.timer_expires;
1002	/*
1003	 * If 'expires' is after the current time, we've been called
1004	 * too early.
1005	 */
1006	if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
1007		dev->power.timer_expires = 0;
1008		rpm_suspend(dev, dev->power.timer_autosuspends ?
1009		    (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
1010	}
1011
1012	spin_unlock_irqrestore(&dev->power.lock, flags);
1013
1014	return HRTIMER_NORESTART;
1015}
1016
1017/**
1018 * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
1019 * @dev: Device to suspend.
1020 * @delay: Time to wait before submitting a suspend request, in milliseconds.
1021 */
1022int pm_schedule_suspend(struct device *dev, unsigned int delay)
1023{
1024	unsigned long flags;
1025	u64 expires;
1026	int retval;
1027
1028	spin_lock_irqsave(&dev->power.lock, flags);
1029
1030	if (!delay) {
1031		retval = rpm_suspend(dev, RPM_ASYNC);
1032		goto out;
1033	}
1034
1035	retval = rpm_check_suspend_allowed(dev);
1036	if (retval)
1037		goto out;
1038
1039	/* Other scheduled or pending requests need to be canceled. */
1040	pm_runtime_cancel_pending(dev);
1041
1042	expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
1043	dev->power.timer_expires = expires;
1044	dev->power.timer_autosuspends = 0;
1045	hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
1046
1047 out:
1048	spin_unlock_irqrestore(&dev->power.lock, flags);
1049
1050	return retval;
1051}
1052EXPORT_SYMBOL_GPL(pm_schedule_suspend);
1053
1054static int rpm_drop_usage_count(struct device *dev)
1055{
1056	int ret;
1057
1058	ret = atomic_sub_return(1, &dev->power.usage_count);
1059	if (ret >= 0)
1060		return ret;
1061
1062	/*
1063	 * Because rpm_resume() does not check the usage counter, it will resume
1064	 * the device even if the usage counter is 0 or negative, so it is
1065	 * sufficient to increment the usage counter here to reverse the change
1066	 * made above.
1067	 */
1068	atomic_inc(&dev->power.usage_count);
1069	dev_warn(dev, "Runtime PM usage count underflow!\n");
1070	return -EINVAL;
1071}
1072
1073/**
1074 * __pm_runtime_idle - Entry point for runtime idle operations.
1075 * @dev: Device to send idle notification for.
1076 * @rpmflags: Flag bits.
1077 *
1078 * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1079 * return immediately if it is larger than zero (if it becomes negative, log a
1080 * warning, increment it, and return an error).  Then carry out an idle
1081 * notification, either synchronous or asynchronous.
1082 *
1083 * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1084 * or if pm_runtime_irq_safe() has been called.
1085 */
1086int __pm_runtime_idle(struct device *dev, int rpmflags)
1087{
1088	unsigned long flags;
1089	int retval;
1090
1091	if (rpmflags & RPM_GET_PUT) {
1092		retval = rpm_drop_usage_count(dev);
1093		if (retval < 0) {
1094			return retval;
1095		} else if (retval > 0) {
1096			trace_rpm_usage(dev, rpmflags);
1097			return 0;
1098		}
1099	}
1100
1101	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1102
1103	spin_lock_irqsave(&dev->power.lock, flags);
1104	retval = rpm_idle(dev, rpmflags);
1105	spin_unlock_irqrestore(&dev->power.lock, flags);
1106
1107	return retval;
1108}
1109EXPORT_SYMBOL_GPL(__pm_runtime_idle);
1110
1111/**
1112 * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
1113 * @dev: Device to suspend.
1114 * @rpmflags: Flag bits.
1115 *
1116 * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1117 * return immediately if it is larger than zero (if it becomes negative, log a
1118 * warning, increment it, and return an error).  Then carry out a suspend,
1119 * either synchronous or asynchronous.
1120 *
1121 * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1122 * or if pm_runtime_irq_safe() has been called.
1123 */
1124int __pm_runtime_suspend(struct device *dev, int rpmflags)
1125{
1126	unsigned long flags;
1127	int retval;
1128
1129	if (rpmflags & RPM_GET_PUT) {
1130		retval = rpm_drop_usage_count(dev);
1131		if (retval < 0) {
1132			return retval;
1133		} else if (retval > 0) {
1134			trace_rpm_usage(dev, rpmflags);
1135			return 0;
1136		}
1137	}
1138
1139	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1140
1141	spin_lock_irqsave(&dev->power.lock, flags);
1142	retval = rpm_suspend(dev, rpmflags);
1143	spin_unlock_irqrestore(&dev->power.lock, flags);
1144
1145	return retval;
1146}
1147EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
1148
1149/**
1150 * __pm_runtime_resume - Entry point for runtime resume operations.
1151 * @dev: Device to resume.
1152 * @rpmflags: Flag bits.
1153 *
1154 * If the RPM_GET_PUT flag is set, increment the device's usage count.  Then
1155 * carry out a resume, either synchronous or asynchronous.
1156 *
1157 * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1158 * or if pm_runtime_irq_safe() has been called.
1159 */
1160int __pm_runtime_resume(struct device *dev, int rpmflags)
1161{
1162	unsigned long flags;
1163	int retval;
1164
1165	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
1166			dev->power.runtime_status != RPM_ACTIVE);
1167
1168	if (rpmflags & RPM_GET_PUT)
1169		atomic_inc(&dev->power.usage_count);
1170
1171	spin_lock_irqsave(&dev->power.lock, flags);
1172	retval = rpm_resume(dev, rpmflags);
1173	spin_unlock_irqrestore(&dev->power.lock, flags);
1174
1175	return retval;
1176}
1177EXPORT_SYMBOL_GPL(__pm_runtime_resume);
1178
1179/**
1180 * pm_runtime_get_conditional - Conditionally bump up device usage counter.
1181 * @dev: Device to handle.
1182 * @ign_usage_count: Whether or not to look at the current usage counter value.
1183 *
1184 * Return -EINVAL if runtime PM is disabled for @dev.
1185 *
1186 * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
1187 * @ign_usage_count is %true or the runtime PM usage counter of @dev is not
1188 * zero, increment the usage counter of @dev and return 1. Otherwise, return 0
1189 * without changing the usage counter.
1190 *
1191 * If @ign_usage_count is %true, this function can be used to prevent suspending
1192 * the device when its runtime PM status is %RPM_ACTIVE.
1193 *
1194 * If @ign_usage_count is %false, this function can be used to prevent
1195 * suspending the device when both its runtime PM status is %RPM_ACTIVE and its
1196 * runtime PM usage counter is not zero.
1197 *
1198 * The caller is responsible for decrementing the runtime PM usage counter of
1199 * @dev after this function has returned a positive value for it.
1200 */
1201static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count)
1202{
1203	unsigned long flags;
1204	int retval;
1205
1206	spin_lock_irqsave(&dev->power.lock, flags);
1207	if (dev->power.disable_depth > 0) {
1208		retval = -EINVAL;
1209	} else if (dev->power.runtime_status != RPM_ACTIVE) {
1210		retval = 0;
1211	} else if (ign_usage_count) {
1212		retval = 1;
1213		atomic_inc(&dev->power.usage_count);
1214	} else {
1215		retval = atomic_inc_not_zero(&dev->power.usage_count);
1216	}
1217	trace_rpm_usage(dev, 0);
1218	spin_unlock_irqrestore(&dev->power.lock, flags);
1219
1220	return retval;
1221}
1222
1223/**
1224 * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is
1225 *			      in active state
1226 * @dev: Target device.
1227 *
1228 * Increment the runtime PM usage counter of @dev if its runtime PM status is
1229 * %RPM_ACTIVE, in which case it returns 1. If the device is in a different
1230 * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the
1231 * device, in which case also the usage_count will remain unmodified.
1232 */
1233int pm_runtime_get_if_active(struct device *dev)
1234{
1235	return pm_runtime_get_conditional(dev, true);
1236}
1237EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
1238
1239/**
1240 * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter.
1241 * @dev: Target device.
1242 *
1243 * Increment the runtime PM usage counter of @dev if its runtime PM status is
1244 * %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case
1245 * it returns 1. If the device is in a different state or its usage_count is 0,
1246 * 0 is returned. -EINVAL is returned if runtime PM is disabled for the device,
1247 * in which case also the usage_count will remain unmodified.
1248 */
1249int pm_runtime_get_if_in_use(struct device *dev)
1250{
1251	return pm_runtime_get_conditional(dev, false);
1252}
1253EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
1254
1255/**
1256 * __pm_runtime_set_status - Set runtime PM status of a device.
1257 * @dev: Device to handle.
1258 * @status: New runtime PM status of the device.
1259 *
1260 * If runtime PM of the device is disabled or its power.runtime_error field is
1261 * different from zero, the status may be changed either to RPM_ACTIVE, or to
1262 * RPM_SUSPENDED, as long as that reflects the actual state of the device.
1263 * However, if the device has a parent and the parent is not active, and the
1264 * parent's power.ignore_children flag is unset, the device's status cannot be
1265 * set to RPM_ACTIVE, so -EBUSY is returned in that case.
1266 *
1267 * If successful, __pm_runtime_set_status() clears the power.runtime_error field
1268 * and the device parent's counter of unsuspended children is modified to
1269 * reflect the new status.  If the new status is RPM_SUSPENDED, an idle
1270 * notification request for the parent is submitted.
1271 *
1272 * If @dev has any suppliers (as reflected by device links to them), and @status
1273 * is RPM_ACTIVE, they will be activated upfront and if the activation of one
1274 * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
1275 * of the @status value) and the suppliers will be deacticated on exit.  The
1276 * error returned by the failing supplier activation will be returned in that
1277 * case.
1278 */
1279int __pm_runtime_set_status(struct device *dev, unsigned int status)
1280{
1281	struct device *parent = dev->parent;
1282	bool notify_parent = false;
1283	unsigned long flags;
1284	int error = 0;
1285
1286	if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
1287		return -EINVAL;
1288
1289	spin_lock_irqsave(&dev->power.lock, flags);
1290
1291	/*
1292	 * Prevent PM-runtime from being enabled for the device or return an
1293	 * error if it is enabled already and working.
1294	 */
1295	if (dev->power.runtime_error || dev->power.disable_depth)
1296		dev->power.disable_depth++;
1297	else
1298		error = -EAGAIN;
1299
1300	spin_unlock_irqrestore(&dev->power.lock, flags);
1301
1302	if (error)
1303		return error;
1304
1305	/*
1306	 * If the new status is RPM_ACTIVE, the suppliers can be activated
1307	 * upfront regardless of the current status, because next time
1308	 * rpm_put_suppliers() runs, the rpm_active refcounts of the links
1309	 * involved will be dropped down to one anyway.
1310	 */
1311	if (status == RPM_ACTIVE) {
1312		int idx = device_links_read_lock();
1313
1314		error = rpm_get_suppliers(dev);
1315		if (error)
1316			status = RPM_SUSPENDED;
1317
1318		device_links_read_unlock(idx);
1319	}
1320
1321	spin_lock_irqsave(&dev->power.lock, flags);
1322
1323	if (dev->power.runtime_status == status || !parent)
1324		goto out_set;
1325
1326	if (status == RPM_SUSPENDED) {
1327		atomic_add_unless(&parent->power.child_count, -1, 0);
1328		notify_parent = !parent->power.ignore_children;
1329	} else {
1330		spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
1331
1332		/*
1333		 * It is invalid to put an active child under a parent that is
1334		 * not active, has runtime PM enabled and the
1335		 * 'power.ignore_children' flag unset.
1336		 */
1337		if (!parent->power.disable_depth &&
1338		    !parent->power.ignore_children &&
1339		    parent->power.runtime_status != RPM_ACTIVE) {
1340			dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
1341				dev_name(dev),
1342				dev_name(parent));
1343			error = -EBUSY;
1344		} else if (dev->power.runtime_status == RPM_SUSPENDED) {
1345			atomic_inc(&parent->power.child_count);
1346		}
1347
1348		spin_unlock(&parent->power.lock);
1349
1350		if (error) {
1351			status = RPM_SUSPENDED;
1352			goto out;
1353		}
1354	}
1355
1356 out_set:
1357	__update_runtime_status(dev, status);
1358	if (!error)
1359		dev->power.runtime_error = 0;
1360
1361 out:
1362	spin_unlock_irqrestore(&dev->power.lock, flags);
1363
1364	if (notify_parent)
1365		pm_request_idle(parent);
1366
1367	if (status == RPM_SUSPENDED) {
1368		int idx = device_links_read_lock();
1369
1370		rpm_put_suppliers(dev);
1371
1372		device_links_read_unlock(idx);
1373	}
1374
1375	pm_runtime_enable(dev);
1376
1377	return error;
1378}
1379EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
1380
1381/**
1382 * __pm_runtime_barrier - Cancel pending requests and wait for completions.
1383 * @dev: Device to handle.
1384 *
1385 * Flush all pending requests for the device from pm_wq and wait for all
1386 * runtime PM operations involving the device in progress to complete.
1387 *
1388 * Should be called under dev->power.lock with interrupts disabled.
1389 */
1390static void __pm_runtime_barrier(struct device *dev)
1391{
1392	pm_runtime_deactivate_timer(dev);
1393
1394	if (dev->power.request_pending) {
1395		dev->power.request = RPM_REQ_NONE;
1396		spin_unlock_irq(&dev->power.lock);
1397
1398		cancel_work_sync(&dev->power.work);
1399
1400		spin_lock_irq(&dev->power.lock);
1401		dev->power.request_pending = false;
1402	}
1403
1404	if (dev->power.runtime_status == RPM_SUSPENDING ||
1405	    dev->power.runtime_status == RPM_RESUMING ||
1406	    dev->power.idle_notification) {
1407		DEFINE_WAIT(wait);
1408
1409		/* Suspend, wake-up or idle notification in progress. */
1410		for (;;) {
1411			prepare_to_wait(&dev->power.wait_queue, &wait,
1412					TASK_UNINTERRUPTIBLE);
1413			if (dev->power.runtime_status != RPM_SUSPENDING
1414			    && dev->power.runtime_status != RPM_RESUMING
1415			    && !dev->power.idle_notification)
1416				break;
1417			spin_unlock_irq(&dev->power.lock);
1418
1419			schedule();
1420
1421			spin_lock_irq(&dev->power.lock);
1422		}
1423		finish_wait(&dev->power.wait_queue, &wait);
1424	}
1425}
1426
1427/**
1428 * pm_runtime_barrier - Flush pending requests and wait for completions.
1429 * @dev: Device to handle.
1430 *
1431 * Prevent the device from being suspended by incrementing its usage counter and
1432 * if there's a pending resume request for the device, wake the device up.
1433 * Next, make sure that all pending requests for the device have been flushed
1434 * from pm_wq and wait for all runtime PM operations involving the device in
1435 * progress to complete.
1436 *
1437 * Return value:
1438 * 1, if there was a resume request pending and the device had to be woken up,
1439 * 0, otherwise
1440 */
1441int pm_runtime_barrier(struct device *dev)
1442{
1443	int retval = 0;
1444
1445	pm_runtime_get_noresume(dev);
1446	spin_lock_irq(&dev->power.lock);
1447
1448	if (dev->power.request_pending
1449	    && dev->power.request == RPM_REQ_RESUME) {
1450		rpm_resume(dev, 0);
1451		retval = 1;
1452	}
1453
1454	__pm_runtime_barrier(dev);
1455
1456	spin_unlock_irq(&dev->power.lock);
1457	pm_runtime_put_noidle(dev);
1458
1459	return retval;
1460}
1461EXPORT_SYMBOL_GPL(pm_runtime_barrier);
1462
1463/**
1464 * __pm_runtime_disable - Disable runtime PM of a device.
1465 * @dev: Device to handle.
1466 * @check_resume: If set, check if there's a resume request for the device.
1467 *
1468 * Increment power.disable_depth for the device and if it was zero previously,
1469 * cancel all pending runtime PM requests for the device and wait for all
1470 * operations in progress to complete.  The device can be either active or
1471 * suspended after its runtime PM has been disabled.
1472 *
1473 * If @check_resume is set and there's a resume request pending when
1474 * __pm_runtime_disable() is called and power.disable_depth is zero, the
1475 * function will wake up the device before disabling its runtime PM.
1476 */
1477void __pm_runtime_disable(struct device *dev, bool check_resume)
1478{
1479	spin_lock_irq(&dev->power.lock);
1480
1481	if (dev->power.disable_depth > 0) {
1482		dev->power.disable_depth++;
1483		goto out;
1484	}
1485
1486	/*
1487	 * Wake up the device if there's a resume request pending, because that
1488	 * means there probably is some I/O to process and disabling runtime PM
1489	 * shouldn't prevent the device from processing the I/O.
1490	 */
1491	if (check_resume && dev->power.request_pending &&
1492	    dev->power.request == RPM_REQ_RESUME) {
1493		/*
1494		 * Prevent suspends and idle notifications from being carried
1495		 * out after we have woken up the device.
1496		 */
1497		pm_runtime_get_noresume(dev);
1498
1499		rpm_resume(dev, 0);
1500
1501		pm_runtime_put_noidle(dev);
1502	}
1503
1504	/* Update time accounting before disabling PM-runtime. */
1505	update_pm_runtime_accounting(dev);
1506
1507	if (!dev->power.disable_depth++) {
1508		__pm_runtime_barrier(dev);
1509		dev->power.last_status = dev->power.runtime_status;
1510	}
1511
1512 out:
1513	spin_unlock_irq(&dev->power.lock);
1514}
1515EXPORT_SYMBOL_GPL(__pm_runtime_disable);
1516
1517/**
1518 * pm_runtime_enable - Enable runtime PM of a device.
1519 * @dev: Device to handle.
1520 */
1521void pm_runtime_enable(struct device *dev)
1522{
1523	unsigned long flags;
1524
1525	spin_lock_irqsave(&dev->power.lock, flags);
1526
1527	if (!dev->power.disable_depth) {
 
 
 
 
 
 
1528		dev_warn(dev, "Unbalanced %s!\n", __func__);
1529		goto out;
1530	}
1531
1532	if (--dev->power.disable_depth > 0)
1533		goto out;
1534
1535	dev->power.last_status = RPM_INVALID;
1536	dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
 
1537
1538	if (dev->power.runtime_status == RPM_SUSPENDED &&
1539	    !dev->power.ignore_children &&
1540	    atomic_read(&dev->power.child_count) > 0)
1541		dev_warn(dev, "Enabling runtime PM for inactive device with active children\n");
1542
1543out:
1544	spin_unlock_irqrestore(&dev->power.lock, flags);
1545}
1546EXPORT_SYMBOL_GPL(pm_runtime_enable);
1547
1548static void pm_runtime_disable_action(void *data)
1549{
1550	pm_runtime_dont_use_autosuspend(data);
1551	pm_runtime_disable(data);
1552}
1553
1554/**
1555 * devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable.
1556 *
1557 * NOTE: this will also handle calling pm_runtime_dont_use_autosuspend() for
1558 * you at driver exit time if needed.
1559 *
1560 * @dev: Device to handle.
1561 */
1562int devm_pm_runtime_enable(struct device *dev)
1563{
1564	pm_runtime_enable(dev);
1565
1566	return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);
1567}
1568EXPORT_SYMBOL_GPL(devm_pm_runtime_enable);
1569
1570/**
1571 * pm_runtime_forbid - Block runtime PM of a device.
1572 * @dev: Device to handle.
1573 *
1574 * Increase the device's usage count and clear its power.runtime_auto flag,
1575 * so that it cannot be suspended at run time until pm_runtime_allow() is called
1576 * for it.
1577 */
1578void pm_runtime_forbid(struct device *dev)
1579{
1580	spin_lock_irq(&dev->power.lock);
1581	if (!dev->power.runtime_auto)
1582		goto out;
1583
1584	dev->power.runtime_auto = false;
1585	atomic_inc(&dev->power.usage_count);
1586	rpm_resume(dev, 0);
1587
1588 out:
1589	spin_unlock_irq(&dev->power.lock);
1590}
1591EXPORT_SYMBOL_GPL(pm_runtime_forbid);
1592
1593/**
1594 * pm_runtime_allow - Unblock runtime PM of a device.
1595 * @dev: Device to handle.
1596 *
1597 * Decrease the device's usage count and set its power.runtime_auto flag.
1598 */
1599void pm_runtime_allow(struct device *dev)
1600{
1601	int ret;
1602
1603	spin_lock_irq(&dev->power.lock);
1604	if (dev->power.runtime_auto)
1605		goto out;
1606
1607	dev->power.runtime_auto = true;
1608	ret = rpm_drop_usage_count(dev);
1609	if (ret == 0)
1610		rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
1611	else if (ret > 0)
1612		trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC);
1613
1614 out:
1615	spin_unlock_irq(&dev->power.lock);
1616}
1617EXPORT_SYMBOL_GPL(pm_runtime_allow);
1618
1619/**
1620 * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
1621 * @dev: Device to handle.
1622 *
1623 * Set the power.no_callbacks flag, which tells the PM core that this
1624 * device is power-managed through its parent and has no runtime PM
1625 * callbacks of its own.  The runtime sysfs attributes will be removed.
1626 */
1627void pm_runtime_no_callbacks(struct device *dev)
1628{
1629	spin_lock_irq(&dev->power.lock);
1630	dev->power.no_callbacks = 1;
1631	spin_unlock_irq(&dev->power.lock);
1632	if (device_is_registered(dev))
1633		rpm_sysfs_remove(dev);
1634}
1635EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
1636
1637/**
1638 * pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
1639 * @dev: Device to handle
1640 *
1641 * Set the power.irq_safe flag, which tells the PM core that the
1642 * ->runtime_suspend() and ->runtime_resume() callbacks for this device should
1643 * always be invoked with the spinlock held and interrupts disabled.  It also
1644 * causes the parent's usage counter to be permanently incremented, preventing
1645 * the parent from runtime suspending -- otherwise an irq-safe child might have
1646 * to wait for a non-irq-safe parent.
1647 */
1648void pm_runtime_irq_safe(struct device *dev)
1649{
1650	if (dev->parent)
1651		pm_runtime_get_sync(dev->parent);
1652
1653	spin_lock_irq(&dev->power.lock);
1654	dev->power.irq_safe = 1;
1655	spin_unlock_irq(&dev->power.lock);
1656}
1657EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
1658
1659/**
1660 * update_autosuspend - Handle a change to a device's autosuspend settings.
1661 * @dev: Device to handle.
1662 * @old_delay: The former autosuspend_delay value.
1663 * @old_use: The former use_autosuspend value.
1664 *
1665 * Prevent runtime suspend if the new delay is negative and use_autosuspend is
1666 * set; otherwise allow it.  Send an idle notification if suspends are allowed.
1667 *
1668 * This function must be called under dev->power.lock with interrupts disabled.
1669 */
1670static void update_autosuspend(struct device *dev, int old_delay, int old_use)
1671{
1672	int delay = dev->power.autosuspend_delay;
1673
1674	/* Should runtime suspend be prevented now? */
1675	if (dev->power.use_autosuspend && delay < 0) {
1676
1677		/* If it used to be allowed then prevent it. */
1678		if (!old_use || old_delay >= 0) {
1679			atomic_inc(&dev->power.usage_count);
1680			rpm_resume(dev, 0);
1681		} else {
1682			trace_rpm_usage(dev, 0);
1683		}
1684	}
1685
1686	/* Runtime suspend should be allowed now. */
1687	else {
1688
1689		/* If it used to be prevented then allow it. */
1690		if (old_use && old_delay < 0)
1691			atomic_dec(&dev->power.usage_count);
1692
1693		/* Maybe we can autosuspend now. */
1694		rpm_idle(dev, RPM_AUTO);
1695	}
1696}
1697
1698/**
1699 * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
1700 * @dev: Device to handle.
1701 * @delay: Value of the new delay in milliseconds.
1702 *
1703 * Set the device's power.autosuspend_delay value.  If it changes to negative
1704 * and the power.use_autosuspend flag is set, prevent runtime suspends.  If it
1705 * changes the other way, allow runtime suspends.
1706 */
1707void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
1708{
1709	int old_delay, old_use;
1710
1711	spin_lock_irq(&dev->power.lock);
1712	old_delay = dev->power.autosuspend_delay;
1713	old_use = dev->power.use_autosuspend;
1714	dev->power.autosuspend_delay = delay;
1715	update_autosuspend(dev, old_delay, old_use);
1716	spin_unlock_irq(&dev->power.lock);
1717}
1718EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
1719
1720/**
1721 * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
1722 * @dev: Device to handle.
1723 * @use: New value for use_autosuspend.
1724 *
1725 * Set the device's power.use_autosuspend flag, and allow or prevent runtime
1726 * suspends as needed.
1727 */
1728void __pm_runtime_use_autosuspend(struct device *dev, bool use)
1729{
1730	int old_delay, old_use;
1731
1732	spin_lock_irq(&dev->power.lock);
1733	old_delay = dev->power.autosuspend_delay;
1734	old_use = dev->power.use_autosuspend;
1735	dev->power.use_autosuspend = use;
1736	update_autosuspend(dev, old_delay, old_use);
1737	spin_unlock_irq(&dev->power.lock);
1738}
1739EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
1740
1741/**
1742 * pm_runtime_init - Initialize runtime PM fields in given device object.
1743 * @dev: Device object to initialize.
1744 */
1745void pm_runtime_init(struct device *dev)
1746{
1747	dev->power.runtime_status = RPM_SUSPENDED;
1748	dev->power.last_status = RPM_INVALID;
1749	dev->power.idle_notification = false;
1750
1751	dev->power.disable_depth = 1;
1752	atomic_set(&dev->power.usage_count, 0);
1753
1754	dev->power.runtime_error = 0;
1755
1756	atomic_set(&dev->power.child_count, 0);
1757	pm_suspend_ignore_children(dev, false);
1758	dev->power.runtime_auto = true;
1759
1760	dev->power.request_pending = false;
1761	dev->power.request = RPM_REQ_NONE;
1762	dev->power.deferred_resume = false;
1763	dev->power.needs_force_resume = 0;
1764	INIT_WORK(&dev->power.work, pm_runtime_work);
1765
1766	dev->power.timer_expires = 0;
1767	hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1768	dev->power.suspend_timer.function = pm_suspend_timer_fn;
1769
1770	init_waitqueue_head(&dev->power.wait_queue);
1771}
1772
1773/**
1774 * pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
1775 * @dev: Device object to re-initialize.
1776 */
1777void pm_runtime_reinit(struct device *dev)
1778{
1779	if (!pm_runtime_enabled(dev)) {
1780		if (dev->power.runtime_status == RPM_ACTIVE)
1781			pm_runtime_set_suspended(dev);
1782		if (dev->power.irq_safe) {
1783			spin_lock_irq(&dev->power.lock);
1784			dev->power.irq_safe = 0;
1785			spin_unlock_irq(&dev->power.lock);
1786			if (dev->parent)
1787				pm_runtime_put(dev->parent);
1788		}
1789	}
1790}
1791
1792/**
1793 * pm_runtime_remove - Prepare for removing a device from device hierarchy.
1794 * @dev: Device object being removed from device hierarchy.
1795 */
1796void pm_runtime_remove(struct device *dev)
1797{
1798	__pm_runtime_disable(dev, false);
1799	pm_runtime_reinit(dev);
1800}
1801
1802/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1803 * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
1804 * @dev: Consumer device.
1805 */
1806void pm_runtime_get_suppliers(struct device *dev)
1807{
1808	struct device_link *link;
1809	int idx;
1810
1811	idx = device_links_read_lock();
1812
1813	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1814				device_links_read_lock_held())
1815		if (link->flags & DL_FLAG_PM_RUNTIME) {
1816			link->supplier_preactivated = true;
 
1817			pm_runtime_get_sync(link->supplier);
1818		}
1819
1820	device_links_read_unlock(idx);
1821}
1822
1823/**
1824 * pm_runtime_put_suppliers - Drop references to supplier devices.
1825 * @dev: Consumer device.
1826 */
1827void pm_runtime_put_suppliers(struct device *dev)
1828{
1829	struct device_link *link;
1830	int idx;
1831
1832	idx = device_links_read_lock();
1833
1834	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1835				device_links_read_lock_held())
1836		if (link->supplier_preactivated) {
1837			link->supplier_preactivated = false;
1838			pm_runtime_put(link->supplier);
 
1839		}
1840
1841	device_links_read_unlock(idx);
1842}
1843
1844void pm_runtime_new_link(struct device *dev)
1845{
1846	spin_lock_irq(&dev->power.lock);
1847	dev->power.links_count++;
1848	spin_unlock_irq(&dev->power.lock);
1849}
1850
1851static void pm_runtime_drop_link_count(struct device *dev)
1852{
1853	spin_lock_irq(&dev->power.lock);
1854	WARN_ON(dev->power.links_count == 0);
1855	dev->power.links_count--;
1856	spin_unlock_irq(&dev->power.lock);
1857}
1858
1859/**
1860 * pm_runtime_drop_link - Prepare for device link removal.
1861 * @link: Device link going away.
1862 *
1863 * Drop the link count of the consumer end of @link and decrement the supplier
1864 * device's runtime PM usage counter as many times as needed to drop all of the
1865 * PM runtime reference to it from the consumer.
1866 */
1867void pm_runtime_drop_link(struct device_link *link)
1868{
1869	if (!(link->flags & DL_FLAG_PM_RUNTIME))
1870		return;
1871
1872	pm_runtime_drop_link_count(link->consumer);
1873	pm_runtime_release_supplier(link);
1874	pm_request_idle(link->supplier);
1875}
1876
1877static bool pm_runtime_need_not_resume(struct device *dev)
1878{
1879	return atomic_read(&dev->power.usage_count) <= 1 &&
1880		(atomic_read(&dev->power.child_count) == 0 ||
1881		 dev->power.ignore_children);
1882}
1883
1884/**
1885 * pm_runtime_force_suspend - Force a device into suspend state if needed.
1886 * @dev: Device to suspend.
1887 *
1888 * Disable runtime PM so we safely can check the device's runtime PM status and
1889 * if it is active, invoke its ->runtime_suspend callback to suspend it and
1890 * change its runtime PM status field to RPM_SUSPENDED.  Also, if the device's
1891 * usage and children counters don't indicate that the device was in use before
1892 * the system-wide transition under way, decrement its parent's children counter
1893 * (if there is a parent).  Keep runtime PM disabled to preserve the state
1894 * unless we encounter errors.
1895 *
1896 * Typically this function may be invoked from a system suspend callback to make
1897 * sure the device is put into low power state and it should only be used during
1898 * system-wide PM transitions to sleep states.  It assumes that the analogous
1899 * pm_runtime_force_resume() will be used to resume the device.
1900 *
1901 * Do not use with DPM_FLAG_SMART_SUSPEND as this can lead to an inconsistent
1902 * state where this function has called the ->runtime_suspend callback but the
1903 * PM core marks the driver as runtime active.
1904 */
1905int pm_runtime_force_suspend(struct device *dev)
1906{
1907	int (*callback)(struct device *);
1908	int ret;
1909
1910	pm_runtime_disable(dev);
1911	if (pm_runtime_status_suspended(dev))
1912		return 0;
1913
1914	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
1915
1916	dev_pm_enable_wake_irq_check(dev, true);
1917	ret = callback ? callback(dev) : 0;
1918	if (ret)
1919		goto err;
1920
1921	dev_pm_enable_wake_irq_complete(dev);
1922
1923	/*
1924	 * If the device can stay in suspend after the system-wide transition
1925	 * to the working state that will follow, drop the children counter of
1926	 * its parent, but set its status to RPM_SUSPENDED anyway in case this
1927	 * function will be called again for it in the meantime.
1928	 */
1929	if (pm_runtime_need_not_resume(dev)) {
1930		pm_runtime_set_suspended(dev);
1931	} else {
1932		__update_runtime_status(dev, RPM_SUSPENDED);
1933		dev->power.needs_force_resume = 1;
1934	}
1935
1936	return 0;
1937
1938err:
1939	dev_pm_disable_wake_irq_check(dev, true);
1940	pm_runtime_enable(dev);
1941	return ret;
1942}
1943EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
1944
1945/**
1946 * pm_runtime_force_resume - Force a device into resume state if needed.
1947 * @dev: Device to resume.
1948 *
1949 * Prior invoking this function we expect the user to have brought the device
1950 * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
1951 * those actions and bring the device into full power, if it is expected to be
1952 * used on system resume.  In the other case, we defer the resume to be managed
1953 * via runtime PM.
1954 *
1955 * Typically this function may be invoked from a system resume callback.
1956 */
1957int pm_runtime_force_resume(struct device *dev)
1958{
1959	int (*callback)(struct device *);
1960	int ret = 0;
1961
1962	if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
1963		goto out;
1964
1965	/*
1966	 * The value of the parent's children counter is correct already, so
1967	 * just update the status of the device.
1968	 */
1969	__update_runtime_status(dev, RPM_ACTIVE);
1970
1971	callback = RPM_GET_CALLBACK(dev, runtime_resume);
1972
1973	dev_pm_disable_wake_irq_check(dev, false);
1974	ret = callback ? callback(dev) : 0;
1975	if (ret) {
1976		pm_runtime_set_suspended(dev);
1977		dev_pm_enable_wake_irq_check(dev, false);
1978		goto out;
1979	}
1980
1981	pm_runtime_mark_last_busy(dev);
1982out:
1983	dev->power.needs_force_resume = 0;
1984	pm_runtime_enable(dev);
1985	return ret;
1986}
1987EXPORT_SYMBOL_GPL(pm_runtime_force_resume);