1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * drivers/base/power/domain_governor.c - Governors for device PM domains.
  4 *
  5 * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
  6 */
  7#include <linux/kernel.h>
  8#include <linux/pm_domain.h>
  9#include <linux/pm_qos.h>
 10#include <linux/hrtimer.h>
 11#include <linux/cpuidle.h>
 12#include <linux/cpumask.h>
 13#include <linux/ktime.h>
 14
 15static int dev_update_qos_constraint(struct device *dev, void *data)
 16{
 17	s64 *constraint_ns_p = data;
 18	s64 constraint_ns;
 19
 20	if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {
 21		struct gpd_timing_data *td = dev_gpd_data(dev)->td;
 22
 23		/*
 24		 * Only take suspend-time QoS constraints of devices into
 25		 * account, because constraints updated after the device has
 26		 * been suspended are not guaranteed to be taken into account
 27		 * anyway.  In order for them to take effect, the device has to
 28		 * be resumed and suspended again.
 29		 */
 30		constraint_ns = td ? td->effective_constraint_ns :
 31				PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
 32	} else {
 33		/*
 34		 * The child is not in a domain and there's no info on its
 35		 * suspend/resume latencies, so assume them to be negligible and
 36		 * take its current PM QoS constraint (that's the only thing
 37		 * known at this point anyway).
 38		 */
 39		constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY);
 40		constraint_ns *= NSEC_PER_USEC;
 41	}
 42
 43	if (constraint_ns < *constraint_ns_p)
 44		*constraint_ns_p = constraint_ns;
 45
 46	return 0;
 47}
 48
 49/**
 50 * default_suspend_ok - Default PM domain governor routine to suspend devices.
 51 * @dev: Device to check.
 52 *
 53 * Returns: true if OK to suspend, false if not OK to suspend
 54 */
 55static bool default_suspend_ok(struct device *dev)
 56{
 57	struct gpd_timing_data *td = dev_gpd_data(dev)->td;
 58	unsigned long flags;
 59	s64 constraint_ns;
 60
 61	dev_dbg(dev, "%s()\n", __func__);
 62
 63	spin_lock_irqsave(&dev->power.lock, flags);
 64
 65	if (!td->constraint_changed) {
 66		bool ret = td->cached_suspend_ok;
 67
 68		spin_unlock_irqrestore(&dev->power.lock, flags);
 69		return ret;
 70	}
 71	td->constraint_changed = false;
 72	td->cached_suspend_ok = false;
 73	td->effective_constraint_ns = 0;
 74	constraint_ns = __dev_pm_qos_resume_latency(dev);
 75
 76	spin_unlock_irqrestore(&dev->power.lock, flags);
 77
 78	if (constraint_ns == 0)
 79		return false;
 80
 81	constraint_ns *= NSEC_PER_USEC;
 82	/*
 83	 * We can walk the children without any additional locking, because
 84	 * they all have been suspended at this point and their
 85	 * effective_constraint_ns fields won't be modified in parallel with us.
 86	 */
 87	if (!dev->power.ignore_children)
 88		device_for_each_child(dev, &constraint_ns,
 89				      dev_update_qos_constraint);
 90
 91	if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) {
 92		/* "No restriction", so the device is allowed to suspend. */
 93		td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
 94		td->cached_suspend_ok = true;
 95	} else if (constraint_ns == 0) {
 96		/*
 97		 * This triggers if one of the children that don't belong to a
 98		 * domain has a zero PM QoS constraint and it's better not to
 99		 * suspend then.  effective_constraint_ns is zero already and
100		 * cached_suspend_ok is false, so bail out.
101		 */
102		return false;
103	} else {
104		constraint_ns -= td->suspend_latency_ns +
105				td->resume_latency_ns;
106		/*
107		 * effective_constraint_ns is zero already and cached_suspend_ok
108		 * is false, so if the computed value is not positive, return
109		 * right away.
110		 */
111		if (constraint_ns <= 0)
112			return false;
113
114		td->effective_constraint_ns = constraint_ns;
115		td->cached_suspend_ok = true;
116	}
117
118	/*
119	 * The children have been suspended already, so we don't need to take
120	 * their suspend latencies into account here.
121	 */
122	return td->cached_suspend_ok;
123}
124
125static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)
126{
127	ktime_t domain_wakeup = KTIME_MAX;
128	ktime_t next_wakeup;
129	struct pm_domain_data *pdd;
130	struct gpd_link *link;
131
132	if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))
133		return;
134
135	/*
136	 * Devices that have a predictable wakeup pattern, may specify
137	 * their next wakeup. Let's find the next wakeup from all the
138	 * devices attached to this domain and from all the sub-domains.
139	 * It is possible that component's a next wakeup may have become
140	 * stale when we read that here. We will ignore to ensure the domain
141	 * is able to enter its optimal idle state.
142	 */
143	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
144		next_wakeup = to_gpd_data(pdd)->td->next_wakeup;
145		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
146			if (ktime_before(next_wakeup, domain_wakeup))
147				domain_wakeup = next_wakeup;
148	}
149
150	list_for_each_entry(link, &genpd->parent_links, parent_node) {
151		struct genpd_governor_data *cgd = link->child->gd;
152
153		next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX;
154		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
155			if (ktime_before(next_wakeup, domain_wakeup))
156				domain_wakeup = next_wakeup;
157	}
158
159	genpd->gd->next_wakeup = domain_wakeup;
160}
161
162static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,
163				     unsigned int state, ktime_t now)
164{
165	ktime_t domain_wakeup = genpd->gd->next_wakeup;
166	s64 idle_time_ns, min_sleep_ns;
167
168	min_sleep_ns = genpd->states[state].power_off_latency_ns +
169		       genpd->states[state].residency_ns;
170
171	idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
172
173	return idle_time_ns >= min_sleep_ns;
174}
175
176static bool __default_power_down_ok(struct dev_pm_domain *pd,
177				     unsigned int state)
178{
179	struct generic_pm_domain *genpd = pd_to_genpd(pd);
180	struct gpd_link *link;
181	struct pm_domain_data *pdd;
182	s64 min_off_time_ns;
183	s64 off_on_time_ns;
184
185	off_on_time_ns = genpd->states[state].power_off_latency_ns +
186		genpd->states[state].power_on_latency_ns;
187
188	min_off_time_ns = -1;
189	/*
190	 * Check if subdomains can be off for enough time.
191	 *
192	 * All subdomains have been powered off already at this point.
193	 */
194	list_for_each_entry(link, &genpd->parent_links, parent_node) {
195		struct genpd_governor_data *cgd = link->child->gd;
196
197		s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1;
198
199		if (sd_max_off_ns < 0)
200			continue;
201
202		/*
203		 * Check if the subdomain is allowed to be off long enough for
204		 * the current domain to turn off and on (that's how much time
205		 * it will have to wait worst case).
206		 */
207		if (sd_max_off_ns <= off_on_time_ns)
208			return false;
209
210		if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0)
211			min_off_time_ns = sd_max_off_ns;
212	}
213
214	/*
215	 * Check if the devices in the domain can be off enough time.
216	 */
217	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
218		struct gpd_timing_data *td;
219		s64 constraint_ns;
220
221		/*
222		 * Check if the device is allowed to be off long enough for the
223		 * domain to turn off and on (that's how much time it will
224		 * have to wait worst case).
225		 */
226		td = to_gpd_data(pdd)->td;
227		constraint_ns = td->effective_constraint_ns;
228		/*
229		 * Zero means "no suspend at all" and this runs only when all
230		 * devices in the domain are suspended, so it must be positive.
231		 */
232		if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS)
233			continue;
234
235		if (constraint_ns <= off_on_time_ns)
236			return false;
237
238		if (min_off_time_ns > constraint_ns || min_off_time_ns < 0)
239			min_off_time_ns = constraint_ns;
240	}
241
242	/*
243	 * If the computed minimum device off time is negative, there are no
244	 * latency constraints, so the domain can spend arbitrary time in the
245	 * "off" state.
246	 */
247	if (min_off_time_ns < 0)
248		return true;
249
250	/*
251	 * The difference between the computed minimum subdomain or device off
252	 * time and the time needed to turn the domain on is the maximum
253	 * theoretical time this domain can spend in the "off" state.
254	 */
255	genpd->gd->max_off_time_ns = min_off_time_ns -
256		genpd->states[state].power_on_latency_ns;
257	return true;
258}
259
260/**
261 * _default_power_down_ok - Default generic PM domain power off governor routine.
262 * @pd: PM domain to check.
263 * @now: current ktime.
264 *
265 * This routine must be executed under the PM domain's lock.
266 *
267 * Returns: true if OK to power down, false if not OK to power down
268 */
269static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)
270{
271	struct generic_pm_domain *genpd = pd_to_genpd(pd);
272	struct genpd_governor_data *gd = genpd->gd;
273	int state_idx = genpd->state_count - 1;
274	struct gpd_link *link;
275
276	/*
277	 * Find the next wakeup from devices that can determine their own wakeup
278	 * to find when the domain would wakeup and do it for every device down
279	 * the hierarchy. It is not worth while to sleep if the state's residency
280	 * cannot be met.
281	 */
282	update_domain_next_wakeup(genpd, now);
283	if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) {
284		/* Let's find out the deepest domain idle state, the devices prefer */
285		while (state_idx >= 0) {
286			if (next_wakeup_allows_state(genpd, state_idx, now)) {
287				gd->max_off_time_changed = true;
288				break;
289			}
290			state_idx--;
291		}
292
293		if (state_idx < 0) {
294			state_idx = 0;
295			gd->cached_power_down_ok = false;
296			goto done;
297		}
298	}
299
300	if (!gd->max_off_time_changed) {
301		genpd->state_idx = gd->cached_power_down_state_idx;
302		return gd->cached_power_down_ok;
303	}
304
305	/*
306	 * We have to invalidate the cached results for the parents, so
307	 * use the observation that default_power_down_ok() is not
308	 * going to be called for any parent until this instance
309	 * returns.
310	 */
311	list_for_each_entry(link, &genpd->child_links, child_node) {
312		struct genpd_governor_data *pgd = link->parent->gd;
313
314		if (pgd)
315			pgd->max_off_time_changed = true;
316	}
317
318	gd->max_off_time_ns = -1;
319	gd->max_off_time_changed = false;
320	gd->cached_power_down_ok = true;
321
322	/*
323	 * Find a state to power down to, starting from the state
324	 * determined by the next wakeup.
325	 */
326	while (!__default_power_down_ok(pd, state_idx)) {
327		if (state_idx == 0) {
328			gd->cached_power_down_ok = false;
329			break;
330		}
331		state_idx--;
332	}
333
334done:
335	genpd->state_idx = state_idx;
336	gd->cached_power_down_state_idx = genpd->state_idx;
337	return gd->cached_power_down_ok;
338}
339
340static bool default_power_down_ok(struct dev_pm_domain *pd)
341{
342	return _default_power_down_ok(pd, ktime_get());
343}
344
345#ifdef CONFIG_CPU_IDLE
346static bool cpu_power_down_ok(struct dev_pm_domain *pd)
347{
348	struct generic_pm_domain *genpd = pd_to_genpd(pd);
349	struct cpuidle_device *dev;
350	ktime_t domain_wakeup, next_hrtimer;
351	ktime_t now = ktime_get();
352	s64 idle_duration_ns;
353	int cpu, i;
354
355	/* Validate dev PM QoS constraints. */
356	if (!_default_power_down_ok(pd, now))
357		return false;
358
359	if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN))
360		return true;
361
362	/*
363	 * Find the next wakeup for any of the online CPUs within the PM domain
364	 * and its subdomains. Note, we only need the genpd->cpus, as it already
365	 * contains a mask of all CPUs from subdomains.
366	 */
367	domain_wakeup = ktime_set(KTIME_SEC_MAX, 0);
368	for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {
369		dev = per_cpu(cpuidle_devices, cpu);
370		if (dev) {
371			next_hrtimer = READ_ONCE(dev->next_hrtimer);
372			if (ktime_before(next_hrtimer, domain_wakeup))
373				domain_wakeup = next_hrtimer;
374		}
375	}
376
377	/* The minimum idle duration is from now - until the next wakeup. */
378	idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
379	if (idle_duration_ns <= 0)
380		return false;
381
382	/* Store the next domain_wakeup to allow consumers to use it. */
383	genpd->gd->next_hrtimer = domain_wakeup;
384
385	/*
386	 * Find the deepest idle state that has its residency value satisfied
387	 * and by also taking into account the power off latency for the state.
388	 * Start at the state picked by the dev PM QoS constraint validation.
389	 */
390	i = genpd->state_idx;
391	do {
392		if (idle_duration_ns >= (genpd->states[i].residency_ns +
393		    genpd->states[i].power_off_latency_ns)) {
394			genpd->state_idx = i;
395			return true;
396		}
397	} while (--i >= 0);
398
399	return false;
400}
401
402struct dev_power_governor pm_domain_cpu_gov = {
403	.suspend_ok = default_suspend_ok,
404	.power_down_ok = cpu_power_down_ok,
405};
406#endif
407
408struct dev_power_governor simple_qos_governor = {
409	.suspend_ok = default_suspend_ok,
410	.power_down_ok = default_power_down_ok,
411};
412
413/*
414 * pm_domain_always_on_gov - A governor implementing an always-on policy
415 */
416struct dev_power_governor pm_domain_always_on_gov = {
417	.suspend_ok = default_suspend_ok,
418};