Loading...
1/*
2 * linux/drivers/cpufreq/cpufreq.c
3 *
4 * Copyright (C) 2001 Russell King
5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6 *
7 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
8 * Added handling for CPU hotplug
9 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
10 * Fix handling for CPU hotplug -- affected CPUs
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
15 *
16 */
17
18#include <linux/kernel.h>
19#include <linux/module.h>
20#include <linux/init.h>
21#include <linux/notifier.h>
22#include <linux/cpufreq.h>
23#include <linux/delay.h>
24#include <linux/interrupt.h>
25#include <linux/spinlock.h>
26#include <linux/device.h>
27#include <linux/slab.h>
28#include <linux/cpu.h>
29#include <linux/completion.h>
30#include <linux/mutex.h>
31#include <linux/syscore_ops.h>
32
33#include <trace/events/power.h>
34
35/**
36 * The "cpufreq driver" - the arch- or hardware-dependent low
37 * level driver of CPUFreq support, and its spinlock. This lock
38 * also protects the cpufreq_cpu_data array.
39 */
40static struct cpufreq_driver *cpufreq_driver;
41static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
42#ifdef CONFIG_HOTPLUG_CPU
43/* This one keeps track of the previously set governor of a removed CPU */
44static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
45#endif
46static DEFINE_SPINLOCK(cpufreq_driver_lock);
47
48/*
49 * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
50 * all cpufreq/hotplug/workqueue/etc related lock issues.
51 *
52 * The rules for this semaphore:
53 * - Any routine that wants to read from the policy structure will
54 * do a down_read on this semaphore.
55 * - Any routine that will write to the policy structure and/or may take away
56 * the policy altogether (eg. CPU hotplug), will hold this lock in write
57 * mode before doing so.
58 *
59 * Additional rules:
60 * - All holders of the lock should check to make sure that the CPU they
61 * are concerned with are online after they get the lock.
62 * - Governor routines that can be called in cpufreq hotplug path should not
63 * take this sem as top level hotplug notifier handler takes this.
64 * - Lock should not be held across
65 * __cpufreq_governor(data, CPUFREQ_GOV_STOP);
66 */
67static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
68static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
69
70#define lock_policy_rwsem(mode, cpu) \
71static int lock_policy_rwsem_##mode \
72(int cpu) \
73{ \
74 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \
75 BUG_ON(policy_cpu == -1); \
76 down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
77 if (unlikely(!cpu_online(cpu))) { \
78 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
79 return -1; \
80 } \
81 \
82 return 0; \
83}
84
85lock_policy_rwsem(read, cpu);
86
87lock_policy_rwsem(write, cpu);
88
89static void unlock_policy_rwsem_read(int cpu)
90{
91 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
92 BUG_ON(policy_cpu == -1);
93 up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
94}
95
96static void unlock_policy_rwsem_write(int cpu)
97{
98 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
99 BUG_ON(policy_cpu == -1);
100 up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
101}
102
103
104/* internal prototypes */
105static int __cpufreq_governor(struct cpufreq_policy *policy,
106 unsigned int event);
107static unsigned int __cpufreq_get(unsigned int cpu);
108static void handle_update(struct work_struct *work);
109
110/**
111 * Two notifier lists: the "policy" list is involved in the
112 * validation process for a new CPU frequency policy; the
113 * "transition" list for kernel code that needs to handle
114 * changes to devices when the CPU clock speed changes.
115 * The mutex locks both lists.
116 */
117static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
118static struct srcu_notifier_head cpufreq_transition_notifier_list;
119
120static bool init_cpufreq_transition_notifier_list_called;
121static int __init init_cpufreq_transition_notifier_list(void)
122{
123 srcu_init_notifier_head(&cpufreq_transition_notifier_list);
124 init_cpufreq_transition_notifier_list_called = true;
125 return 0;
126}
127pure_initcall(init_cpufreq_transition_notifier_list);
128
129static LIST_HEAD(cpufreq_governor_list);
130static DEFINE_MUTEX(cpufreq_governor_mutex);
131
132struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
133{
134 struct cpufreq_policy *data;
135 unsigned long flags;
136
137 if (cpu >= nr_cpu_ids)
138 goto err_out;
139
140 /* get the cpufreq driver */
141 spin_lock_irqsave(&cpufreq_driver_lock, flags);
142
143 if (!cpufreq_driver)
144 goto err_out_unlock;
145
146 if (!try_module_get(cpufreq_driver->owner))
147 goto err_out_unlock;
148
149
150 /* get the CPU */
151 data = per_cpu(cpufreq_cpu_data, cpu);
152
153 if (!data)
154 goto err_out_put_module;
155
156 if (!kobject_get(&data->kobj))
157 goto err_out_put_module;
158
159 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
160 return data;
161
162err_out_put_module:
163 module_put(cpufreq_driver->owner);
164err_out_unlock:
165 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
166err_out:
167 return NULL;
168}
169EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
170
171
172void cpufreq_cpu_put(struct cpufreq_policy *data)
173{
174 kobject_put(&data->kobj);
175 module_put(cpufreq_driver->owner);
176}
177EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
178
179
180/*********************************************************************
181 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
182 *********************************************************************/
183
184/**
185 * adjust_jiffies - adjust the system "loops_per_jiffy"
186 *
187 * This function alters the system "loops_per_jiffy" for the clock
188 * speed change. Note that loops_per_jiffy cannot be updated on SMP
189 * systems as each CPU might be scaled differently. So, use the arch
190 * per-CPU loops_per_jiffy value wherever possible.
191 */
192#ifndef CONFIG_SMP
193static unsigned long l_p_j_ref;
194static unsigned int l_p_j_ref_freq;
195
196static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
197{
198 if (ci->flags & CPUFREQ_CONST_LOOPS)
199 return;
200
201 if (!l_p_j_ref_freq) {
202 l_p_j_ref = loops_per_jiffy;
203 l_p_j_ref_freq = ci->old;
204 pr_debug("saving %lu as reference value for loops_per_jiffy; "
205 "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
206 }
207 if ((val == CPUFREQ_PRECHANGE && ci->old < ci->new) ||
208 (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
209 (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
210 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
211 ci->new);
212 pr_debug("scaling loops_per_jiffy to %lu "
213 "for frequency %u kHz\n", loops_per_jiffy, ci->new);
214 }
215}
216#else
217static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
218{
219 return;
220}
221#endif
222
223
224/**
225 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
226 * on frequency transition.
227 *
228 * This function calls the transition notifiers and the "adjust_jiffies"
229 * function. It is called twice on all CPU frequency changes that have
230 * external effects.
231 */
232void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
233{
234 struct cpufreq_policy *policy;
235
236 BUG_ON(irqs_disabled());
237
238 freqs->flags = cpufreq_driver->flags;
239 pr_debug("notification %u of frequency transition to %u kHz\n",
240 state, freqs->new);
241
242 policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
243 switch (state) {
244
245 case CPUFREQ_PRECHANGE:
246 /* detect if the driver reported a value as "old frequency"
247 * which is not equal to what the cpufreq core thinks is
248 * "old frequency".
249 */
250 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
251 if ((policy) && (policy->cpu == freqs->cpu) &&
252 (policy->cur) && (policy->cur != freqs->old)) {
253 pr_debug("Warning: CPU frequency is"
254 " %u, cpufreq assumed %u kHz.\n",
255 freqs->old, policy->cur);
256 freqs->old = policy->cur;
257 }
258 }
259 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
260 CPUFREQ_PRECHANGE, freqs);
261 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
262 break;
263
264 case CPUFREQ_POSTCHANGE:
265 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
266 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
267 (unsigned long)freqs->cpu);
268 trace_power_frequency(POWER_PSTATE, freqs->new, freqs->cpu);
269 trace_cpu_frequency(freqs->new, freqs->cpu);
270 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
271 CPUFREQ_POSTCHANGE, freqs);
272 if (likely(policy) && likely(policy->cpu == freqs->cpu))
273 policy->cur = freqs->new;
274 break;
275 }
276}
277EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
278
279
280
281/*********************************************************************
282 * SYSFS INTERFACE *
283 *********************************************************************/
284
285static struct cpufreq_governor *__find_governor(const char *str_governor)
286{
287 struct cpufreq_governor *t;
288
289 list_for_each_entry(t, &cpufreq_governor_list, governor_list)
290 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
291 return t;
292
293 return NULL;
294}
295
296/**
297 * cpufreq_parse_governor - parse a governor string
298 */
299static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
300 struct cpufreq_governor **governor)
301{
302 int err = -EINVAL;
303
304 if (!cpufreq_driver)
305 goto out;
306
307 if (cpufreq_driver->setpolicy) {
308 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
309 *policy = CPUFREQ_POLICY_PERFORMANCE;
310 err = 0;
311 } else if (!strnicmp(str_governor, "powersave",
312 CPUFREQ_NAME_LEN)) {
313 *policy = CPUFREQ_POLICY_POWERSAVE;
314 err = 0;
315 }
316 } else if (cpufreq_driver->target) {
317 struct cpufreq_governor *t;
318
319 mutex_lock(&cpufreq_governor_mutex);
320
321 t = __find_governor(str_governor);
322
323 if (t == NULL) {
324 int ret;
325
326 mutex_unlock(&cpufreq_governor_mutex);
327 ret = request_module("cpufreq_%s", str_governor);
328 mutex_lock(&cpufreq_governor_mutex);
329
330 if (ret == 0)
331 t = __find_governor(str_governor);
332 }
333
334 if (t != NULL) {
335 *governor = t;
336 err = 0;
337 }
338
339 mutex_unlock(&cpufreq_governor_mutex);
340 }
341out:
342 return err;
343}
344
345
346/**
347 * cpufreq_per_cpu_attr_read() / show_##file_name() -
348 * print out cpufreq information
349 *
350 * Write out information from cpufreq_driver->policy[cpu]; object must be
351 * "unsigned int".
352 */
353
354#define show_one(file_name, object) \
355static ssize_t show_##file_name \
356(struct cpufreq_policy *policy, char *buf) \
357{ \
358 return sprintf(buf, "%u\n", policy->object); \
359}
360
361show_one(cpuinfo_min_freq, cpuinfo.min_freq);
362show_one(cpuinfo_max_freq, cpuinfo.max_freq);
363show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
364show_one(scaling_min_freq, min);
365show_one(scaling_max_freq, max);
366show_one(scaling_cur_freq, cur);
367
368static int __cpufreq_set_policy(struct cpufreq_policy *data,
369 struct cpufreq_policy *policy);
370
371/**
372 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
373 */
374#define store_one(file_name, object) \
375static ssize_t store_##file_name \
376(struct cpufreq_policy *policy, const char *buf, size_t count) \
377{ \
378 unsigned int ret = -EINVAL; \
379 struct cpufreq_policy new_policy; \
380 \
381 ret = cpufreq_get_policy(&new_policy, policy->cpu); \
382 if (ret) \
383 return -EINVAL; \
384 \
385 ret = sscanf(buf, "%u", &new_policy.object); \
386 if (ret != 1) \
387 return -EINVAL; \
388 \
389 ret = __cpufreq_set_policy(policy, &new_policy); \
390 policy->user_policy.object = policy->object; \
391 \
392 return ret ? ret : count; \
393}
394
395store_one(scaling_min_freq, min);
396store_one(scaling_max_freq, max);
397
398/**
399 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
400 */
401static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
402 char *buf)
403{
404 unsigned int cur_freq = __cpufreq_get(policy->cpu);
405 if (!cur_freq)
406 return sprintf(buf, "<unknown>");
407 return sprintf(buf, "%u\n", cur_freq);
408}
409
410
411/**
412 * show_scaling_governor - show the current policy for the specified CPU
413 */
414static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
415{
416 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
417 return sprintf(buf, "powersave\n");
418 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
419 return sprintf(buf, "performance\n");
420 else if (policy->governor)
421 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
422 policy->governor->name);
423 return -EINVAL;
424}
425
426
427/**
428 * store_scaling_governor - store policy for the specified CPU
429 */
430static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
431 const char *buf, size_t count)
432{
433 unsigned int ret = -EINVAL;
434 char str_governor[16];
435 struct cpufreq_policy new_policy;
436
437 ret = cpufreq_get_policy(&new_policy, policy->cpu);
438 if (ret)
439 return ret;
440
441 ret = sscanf(buf, "%15s", str_governor);
442 if (ret != 1)
443 return -EINVAL;
444
445 if (cpufreq_parse_governor(str_governor, &new_policy.policy,
446 &new_policy.governor))
447 return -EINVAL;
448
449 /* Do not use cpufreq_set_policy here or the user_policy.max
450 will be wrongly overridden */
451 ret = __cpufreq_set_policy(policy, &new_policy);
452
453 policy->user_policy.policy = policy->policy;
454 policy->user_policy.governor = policy->governor;
455
456 if (ret)
457 return ret;
458 else
459 return count;
460}
461
462/**
463 * show_scaling_driver - show the cpufreq driver currently loaded
464 */
465static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
466{
467 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
468}
469
470/**
471 * show_scaling_available_governors - show the available CPUfreq governors
472 */
473static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
474 char *buf)
475{
476 ssize_t i = 0;
477 struct cpufreq_governor *t;
478
479 if (!cpufreq_driver->target) {
480 i += sprintf(buf, "performance powersave");
481 goto out;
482 }
483
484 list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
485 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
486 - (CPUFREQ_NAME_LEN + 2)))
487 goto out;
488 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
489 }
490out:
491 i += sprintf(&buf[i], "\n");
492 return i;
493}
494
495static ssize_t show_cpus(const struct cpumask *mask, char *buf)
496{
497 ssize_t i = 0;
498 unsigned int cpu;
499
500 for_each_cpu(cpu, mask) {
501 if (i)
502 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
503 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
504 if (i >= (PAGE_SIZE - 5))
505 break;
506 }
507 i += sprintf(&buf[i], "\n");
508 return i;
509}
510
511/**
512 * show_related_cpus - show the CPUs affected by each transition even if
513 * hw coordination is in use
514 */
515static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
516{
517 if (cpumask_empty(policy->related_cpus))
518 return show_cpus(policy->cpus, buf);
519 return show_cpus(policy->related_cpus, buf);
520}
521
522/**
523 * show_affected_cpus - show the CPUs affected by each transition
524 */
525static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
526{
527 return show_cpus(policy->cpus, buf);
528}
529
530static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
531 const char *buf, size_t count)
532{
533 unsigned int freq = 0;
534 unsigned int ret;
535
536 if (!policy->governor || !policy->governor->store_setspeed)
537 return -EINVAL;
538
539 ret = sscanf(buf, "%u", &freq);
540 if (ret != 1)
541 return -EINVAL;
542
543 policy->governor->store_setspeed(policy, freq);
544
545 return count;
546}
547
548static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
549{
550 if (!policy->governor || !policy->governor->show_setspeed)
551 return sprintf(buf, "<unsupported>\n");
552
553 return policy->governor->show_setspeed(policy, buf);
554}
555
556/**
557 * show_scaling_driver - show the current cpufreq HW/BIOS limitation
558 */
559static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
560{
561 unsigned int limit;
562 int ret;
563 if (cpufreq_driver->bios_limit) {
564 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
565 if (!ret)
566 return sprintf(buf, "%u\n", limit);
567 }
568 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
569}
570
571cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
572cpufreq_freq_attr_ro(cpuinfo_min_freq);
573cpufreq_freq_attr_ro(cpuinfo_max_freq);
574cpufreq_freq_attr_ro(cpuinfo_transition_latency);
575cpufreq_freq_attr_ro(scaling_available_governors);
576cpufreq_freq_attr_ro(scaling_driver);
577cpufreq_freq_attr_ro(scaling_cur_freq);
578cpufreq_freq_attr_ro(bios_limit);
579cpufreq_freq_attr_ro(related_cpus);
580cpufreq_freq_attr_ro(affected_cpus);
581cpufreq_freq_attr_rw(scaling_min_freq);
582cpufreq_freq_attr_rw(scaling_max_freq);
583cpufreq_freq_attr_rw(scaling_governor);
584cpufreq_freq_attr_rw(scaling_setspeed);
585
586static struct attribute *default_attrs[] = {
587 &cpuinfo_min_freq.attr,
588 &cpuinfo_max_freq.attr,
589 &cpuinfo_transition_latency.attr,
590 &scaling_min_freq.attr,
591 &scaling_max_freq.attr,
592 &affected_cpus.attr,
593 &related_cpus.attr,
594 &scaling_governor.attr,
595 &scaling_driver.attr,
596 &scaling_available_governors.attr,
597 &scaling_setspeed.attr,
598 NULL
599};
600
601struct kobject *cpufreq_global_kobject;
602EXPORT_SYMBOL(cpufreq_global_kobject);
603
604#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
605#define to_attr(a) container_of(a, struct freq_attr, attr)
606
607static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
608{
609 struct cpufreq_policy *policy = to_policy(kobj);
610 struct freq_attr *fattr = to_attr(attr);
611 ssize_t ret = -EINVAL;
612 policy = cpufreq_cpu_get(policy->cpu);
613 if (!policy)
614 goto no_policy;
615
616 if (lock_policy_rwsem_read(policy->cpu) < 0)
617 goto fail;
618
619 if (fattr->show)
620 ret = fattr->show(policy, buf);
621 else
622 ret = -EIO;
623
624 unlock_policy_rwsem_read(policy->cpu);
625fail:
626 cpufreq_cpu_put(policy);
627no_policy:
628 return ret;
629}
630
631static ssize_t store(struct kobject *kobj, struct attribute *attr,
632 const char *buf, size_t count)
633{
634 struct cpufreq_policy *policy = to_policy(kobj);
635 struct freq_attr *fattr = to_attr(attr);
636 ssize_t ret = -EINVAL;
637 policy = cpufreq_cpu_get(policy->cpu);
638 if (!policy)
639 goto no_policy;
640
641 if (lock_policy_rwsem_write(policy->cpu) < 0)
642 goto fail;
643
644 if (fattr->store)
645 ret = fattr->store(policy, buf, count);
646 else
647 ret = -EIO;
648
649 unlock_policy_rwsem_write(policy->cpu);
650fail:
651 cpufreq_cpu_put(policy);
652no_policy:
653 return ret;
654}
655
656static void cpufreq_sysfs_release(struct kobject *kobj)
657{
658 struct cpufreq_policy *policy = to_policy(kobj);
659 pr_debug("last reference is dropped\n");
660 complete(&policy->kobj_unregister);
661}
662
663static const struct sysfs_ops sysfs_ops = {
664 .show = show,
665 .store = store,
666};
667
668static struct kobj_type ktype_cpufreq = {
669 .sysfs_ops = &sysfs_ops,
670 .default_attrs = default_attrs,
671 .release = cpufreq_sysfs_release,
672};
673
674/*
675 * Returns:
676 * Negative: Failure
677 * 0: Success
678 * Positive: When we have a managed CPU and the sysfs got symlinked
679 */
680static int cpufreq_add_dev_policy(unsigned int cpu,
681 struct cpufreq_policy *policy,
682 struct sys_device *sys_dev)
683{
684 int ret = 0;
685#ifdef CONFIG_SMP
686 unsigned long flags;
687 unsigned int j;
688#ifdef CONFIG_HOTPLUG_CPU
689 struct cpufreq_governor *gov;
690
691 gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
692 if (gov) {
693 policy->governor = gov;
694 pr_debug("Restoring governor %s for cpu %d\n",
695 policy->governor->name, cpu);
696 }
697#endif
698
699 for_each_cpu(j, policy->cpus) {
700 struct cpufreq_policy *managed_policy;
701
702 if (cpu == j)
703 continue;
704
705 /* Check for existing affected CPUs.
706 * They may not be aware of it due to CPU Hotplug.
707 * cpufreq_cpu_put is called when the device is removed
708 * in __cpufreq_remove_dev()
709 */
710 managed_policy = cpufreq_cpu_get(j);
711 if (unlikely(managed_policy)) {
712
713 /* Set proper policy_cpu */
714 unlock_policy_rwsem_write(cpu);
715 per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu;
716
717 if (lock_policy_rwsem_write(cpu) < 0) {
718 /* Should not go through policy unlock path */
719 if (cpufreq_driver->exit)
720 cpufreq_driver->exit(policy);
721 cpufreq_cpu_put(managed_policy);
722 return -EBUSY;
723 }
724
725 spin_lock_irqsave(&cpufreq_driver_lock, flags);
726 cpumask_copy(managed_policy->cpus, policy->cpus);
727 per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
728 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
729
730 pr_debug("CPU already managed, adding link\n");
731 ret = sysfs_create_link(&sys_dev->kobj,
732 &managed_policy->kobj,
733 "cpufreq");
734 if (ret)
735 cpufreq_cpu_put(managed_policy);
736 /*
737 * Success. We only needed to be added to the mask.
738 * Call driver->exit() because only the cpu parent of
739 * the kobj needed to call init().
740 */
741 if (cpufreq_driver->exit)
742 cpufreq_driver->exit(policy);
743
744 if (!ret)
745 return 1;
746 else
747 return ret;
748 }
749 }
750#endif
751 return ret;
752}
753
754
755/* symlink affected CPUs */
756static int cpufreq_add_dev_symlink(unsigned int cpu,
757 struct cpufreq_policy *policy)
758{
759 unsigned int j;
760 int ret = 0;
761
762 for_each_cpu(j, policy->cpus) {
763 struct cpufreq_policy *managed_policy;
764 struct sys_device *cpu_sys_dev;
765
766 if (j == cpu)
767 continue;
768 if (!cpu_online(j))
769 continue;
770
771 pr_debug("CPU %u already managed, adding link\n", j);
772 managed_policy = cpufreq_cpu_get(cpu);
773 cpu_sys_dev = get_cpu_sysdev(j);
774 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
775 "cpufreq");
776 if (ret) {
777 cpufreq_cpu_put(managed_policy);
778 return ret;
779 }
780 }
781 return ret;
782}
783
784static int cpufreq_add_dev_interface(unsigned int cpu,
785 struct cpufreq_policy *policy,
786 struct sys_device *sys_dev)
787{
788 struct cpufreq_policy new_policy;
789 struct freq_attr **drv_attr;
790 unsigned long flags;
791 int ret = 0;
792 unsigned int j;
793
794 /* prepare interface data */
795 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
796 &sys_dev->kobj, "cpufreq");
797 if (ret)
798 return ret;
799
800 /* set up files for this cpu device */
801 drv_attr = cpufreq_driver->attr;
802 while ((drv_attr) && (*drv_attr)) {
803 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
804 if (ret)
805 goto err_out_kobj_put;
806 drv_attr++;
807 }
808 if (cpufreq_driver->get) {
809 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
810 if (ret)
811 goto err_out_kobj_put;
812 }
813 if (cpufreq_driver->target) {
814 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
815 if (ret)
816 goto err_out_kobj_put;
817 }
818 if (cpufreq_driver->bios_limit) {
819 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
820 if (ret)
821 goto err_out_kobj_put;
822 }
823
824 spin_lock_irqsave(&cpufreq_driver_lock, flags);
825 for_each_cpu(j, policy->cpus) {
826 if (!cpu_online(j))
827 continue;
828 per_cpu(cpufreq_cpu_data, j) = policy;
829 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
830 }
831 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
832
833 ret = cpufreq_add_dev_symlink(cpu, policy);
834 if (ret)
835 goto err_out_kobj_put;
836
837 memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
838 /* assure that the starting sequence is run in __cpufreq_set_policy */
839 policy->governor = NULL;
840
841 /* set default policy */
842 ret = __cpufreq_set_policy(policy, &new_policy);
843 policy->user_policy.policy = policy->policy;
844 policy->user_policy.governor = policy->governor;
845
846 if (ret) {
847 pr_debug("setting policy failed\n");
848 if (cpufreq_driver->exit)
849 cpufreq_driver->exit(policy);
850 }
851 return ret;
852
853err_out_kobj_put:
854 kobject_put(&policy->kobj);
855 wait_for_completion(&policy->kobj_unregister);
856 return ret;
857}
858
859
860/**
861 * cpufreq_add_dev - add a CPU device
862 *
863 * Adds the cpufreq interface for a CPU device.
864 *
865 * The Oracle says: try running cpufreq registration/unregistration concurrently
866 * with with cpu hotplugging and all hell will break loose. Tried to clean this
867 * mess up, but more thorough testing is needed. - Mathieu
868 */
869static int cpufreq_add_dev(struct sys_device *sys_dev)
870{
871 unsigned int cpu = sys_dev->id;
872 int ret = 0, found = 0;
873 struct cpufreq_policy *policy;
874 unsigned long flags;
875 unsigned int j;
876#ifdef CONFIG_HOTPLUG_CPU
877 int sibling;
878#endif
879
880 if (cpu_is_offline(cpu))
881 return 0;
882
883 pr_debug("adding CPU %u\n", cpu);
884
885#ifdef CONFIG_SMP
886 /* check whether a different CPU already registered this
887 * CPU because it is in the same boat. */
888 policy = cpufreq_cpu_get(cpu);
889 if (unlikely(policy)) {
890 cpufreq_cpu_put(policy);
891 return 0;
892 }
893#endif
894
895 if (!try_module_get(cpufreq_driver->owner)) {
896 ret = -EINVAL;
897 goto module_out;
898 }
899
900 ret = -ENOMEM;
901 policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
902 if (!policy)
903 goto nomem_out;
904
905 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
906 goto err_free_policy;
907
908 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
909 goto err_free_cpumask;
910
911 policy->cpu = cpu;
912 cpumask_copy(policy->cpus, cpumask_of(cpu));
913
914 /* Initially set CPU itself as the policy_cpu */
915 per_cpu(cpufreq_policy_cpu, cpu) = cpu;
916 ret = (lock_policy_rwsem_write(cpu) < 0);
917 WARN_ON(ret);
918
919 init_completion(&policy->kobj_unregister);
920 INIT_WORK(&policy->update, handle_update);
921
922 /* Set governor before ->init, so that driver could check it */
923#ifdef CONFIG_HOTPLUG_CPU
924 for_each_online_cpu(sibling) {
925 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
926 if (cp && cp->governor &&
927 (cpumask_test_cpu(cpu, cp->related_cpus))) {
928 policy->governor = cp->governor;
929 found = 1;
930 break;
931 }
932 }
933#endif
934 if (!found)
935 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
936 /* call driver. From then on the cpufreq must be able
937 * to accept all calls to ->verify and ->setpolicy for this CPU
938 */
939 ret = cpufreq_driver->init(policy);
940 if (ret) {
941 pr_debug("initialization failed\n");
942 goto err_unlock_policy;
943 }
944 policy->user_policy.min = policy->min;
945 policy->user_policy.max = policy->max;
946
947 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
948 CPUFREQ_START, policy);
949
950 ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
951 if (ret) {
952 if (ret > 0)
953 /* This is a managed cpu, symlink created,
954 exit with 0 */
955 ret = 0;
956 goto err_unlock_policy;
957 }
958
959 ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
960 if (ret)
961 goto err_out_unregister;
962
963 unlock_policy_rwsem_write(cpu);
964
965 kobject_uevent(&policy->kobj, KOBJ_ADD);
966 module_put(cpufreq_driver->owner);
967 pr_debug("initialization complete\n");
968
969 return 0;
970
971
972err_out_unregister:
973 spin_lock_irqsave(&cpufreq_driver_lock, flags);
974 for_each_cpu(j, policy->cpus)
975 per_cpu(cpufreq_cpu_data, j) = NULL;
976 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
977
978 kobject_put(&policy->kobj);
979 wait_for_completion(&policy->kobj_unregister);
980
981err_unlock_policy:
982 unlock_policy_rwsem_write(cpu);
983 free_cpumask_var(policy->related_cpus);
984err_free_cpumask:
985 free_cpumask_var(policy->cpus);
986err_free_policy:
987 kfree(policy);
988nomem_out:
989 module_put(cpufreq_driver->owner);
990module_out:
991 return ret;
992}
993
994
995/**
996 * __cpufreq_remove_dev - remove a CPU device
997 *
998 * Removes the cpufreq interface for a CPU device.
999 * Caller should already have policy_rwsem in write mode for this CPU.
1000 * This routine frees the rwsem before returning.
1001 */
1002static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1003{
1004 unsigned int cpu = sys_dev->id;
1005 unsigned long flags;
1006 struct cpufreq_policy *data;
1007 struct kobject *kobj;
1008 struct completion *cmp;
1009#ifdef CONFIG_SMP
1010 struct sys_device *cpu_sys_dev;
1011 unsigned int j;
1012#endif
1013
1014 pr_debug("unregistering CPU %u\n", cpu);
1015
1016 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1017 data = per_cpu(cpufreq_cpu_data, cpu);
1018
1019 if (!data) {
1020 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1021 unlock_policy_rwsem_write(cpu);
1022 return -EINVAL;
1023 }
1024 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1025
1026
1027#ifdef CONFIG_SMP
1028 /* if this isn't the CPU which is the parent of the kobj, we
1029 * only need to unlink, put and exit
1030 */
1031 if (unlikely(cpu != data->cpu)) {
1032 pr_debug("removing link\n");
1033 cpumask_clear_cpu(cpu, data->cpus);
1034 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1035 kobj = &sys_dev->kobj;
1036 cpufreq_cpu_put(data);
1037 unlock_policy_rwsem_write(cpu);
1038 sysfs_remove_link(kobj, "cpufreq");
1039 return 0;
1040 }
1041#endif
1042
1043#ifdef CONFIG_SMP
1044
1045#ifdef CONFIG_HOTPLUG_CPU
1046 strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1047 CPUFREQ_NAME_LEN);
1048#endif
1049
1050 /* if we have other CPUs still registered, we need to unlink them,
1051 * or else wait_for_completion below will lock up. Clean the
1052 * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1053 * the sysfs links afterwards.
1054 */
1055 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1056 for_each_cpu(j, data->cpus) {
1057 if (j == cpu)
1058 continue;
1059 per_cpu(cpufreq_cpu_data, j) = NULL;
1060 }
1061 }
1062
1063 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1064
1065 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1066 for_each_cpu(j, data->cpus) {
1067 if (j == cpu)
1068 continue;
1069 pr_debug("removing link for cpu %u\n", j);
1070#ifdef CONFIG_HOTPLUG_CPU
1071 strncpy(per_cpu(cpufreq_cpu_governor, j),
1072 data->governor->name, CPUFREQ_NAME_LEN);
1073#endif
1074 cpu_sys_dev = get_cpu_sysdev(j);
1075 kobj = &cpu_sys_dev->kobj;
1076 unlock_policy_rwsem_write(cpu);
1077 sysfs_remove_link(kobj, "cpufreq");
1078 lock_policy_rwsem_write(cpu);
1079 cpufreq_cpu_put(data);
1080 }
1081 }
1082#else
1083 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1084#endif
1085
1086 if (cpufreq_driver->target)
1087 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1088
1089 kobj = &data->kobj;
1090 cmp = &data->kobj_unregister;
1091 unlock_policy_rwsem_write(cpu);
1092 kobject_put(kobj);
1093
1094 /* we need to make sure that the underlying kobj is actually
1095 * not referenced anymore by anybody before we proceed with
1096 * unloading.
1097 */
1098 pr_debug("waiting for dropping of refcount\n");
1099 wait_for_completion(cmp);
1100 pr_debug("wait complete\n");
1101
1102 lock_policy_rwsem_write(cpu);
1103 if (cpufreq_driver->exit)
1104 cpufreq_driver->exit(data);
1105 unlock_policy_rwsem_write(cpu);
1106
1107#ifdef CONFIG_HOTPLUG_CPU
1108 /* when the CPU which is the parent of the kobj is hotplugged
1109 * offline, check for siblings, and create cpufreq sysfs interface
1110 * and symlinks
1111 */
1112 if (unlikely(cpumask_weight(data->cpus) > 1)) {
1113 /* first sibling now owns the new sysfs dir */
1114 cpumask_clear_cpu(cpu, data->cpus);
1115 cpufreq_add_dev(get_cpu_sysdev(cpumask_first(data->cpus)));
1116
1117 /* finally remove our own symlink */
1118 lock_policy_rwsem_write(cpu);
1119 __cpufreq_remove_dev(sys_dev);
1120 }
1121#endif
1122
1123 free_cpumask_var(data->related_cpus);
1124 free_cpumask_var(data->cpus);
1125 kfree(data);
1126
1127 return 0;
1128}
1129
1130
1131static int cpufreq_remove_dev(struct sys_device *sys_dev)
1132{
1133 unsigned int cpu = sys_dev->id;
1134 int retval;
1135
1136 if (cpu_is_offline(cpu))
1137 return 0;
1138
1139 if (unlikely(lock_policy_rwsem_write(cpu)))
1140 BUG();
1141
1142 retval = __cpufreq_remove_dev(sys_dev);
1143 return retval;
1144}
1145
1146
1147static void handle_update(struct work_struct *work)
1148{
1149 struct cpufreq_policy *policy =
1150 container_of(work, struct cpufreq_policy, update);
1151 unsigned int cpu = policy->cpu;
1152 pr_debug("handle_update for cpu %u called\n", cpu);
1153 cpufreq_update_policy(cpu);
1154}
1155
1156/**
1157 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1158 * @cpu: cpu number
1159 * @old_freq: CPU frequency the kernel thinks the CPU runs at
1160 * @new_freq: CPU frequency the CPU actually runs at
1161 *
1162 * We adjust to current frequency first, and need to clean up later.
1163 * So either call to cpufreq_update_policy() or schedule handle_update()).
1164 */
1165static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1166 unsigned int new_freq)
1167{
1168 struct cpufreq_freqs freqs;
1169
1170 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1171 "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1172
1173 freqs.cpu = cpu;
1174 freqs.old = old_freq;
1175 freqs.new = new_freq;
1176 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1177 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1178}
1179
1180
1181/**
1182 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1183 * @cpu: CPU number
1184 *
1185 * This is the last known freq, without actually getting it from the driver.
1186 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1187 */
1188unsigned int cpufreq_quick_get(unsigned int cpu)
1189{
1190 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1191 unsigned int ret_freq = 0;
1192
1193 if (policy) {
1194 ret_freq = policy->cur;
1195 cpufreq_cpu_put(policy);
1196 }
1197
1198 return ret_freq;
1199}
1200EXPORT_SYMBOL(cpufreq_quick_get);
1201
1202/**
1203 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1204 * @cpu: CPU number
1205 *
1206 * Just return the max possible frequency for a given CPU.
1207 */
1208unsigned int cpufreq_quick_get_max(unsigned int cpu)
1209{
1210 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1211 unsigned int ret_freq = 0;
1212
1213 if (policy) {
1214 ret_freq = policy->max;
1215 cpufreq_cpu_put(policy);
1216 }
1217
1218 return ret_freq;
1219}
1220EXPORT_SYMBOL(cpufreq_quick_get_max);
1221
1222
1223static unsigned int __cpufreq_get(unsigned int cpu)
1224{
1225 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1226 unsigned int ret_freq = 0;
1227
1228 if (!cpufreq_driver->get)
1229 return ret_freq;
1230
1231 ret_freq = cpufreq_driver->get(cpu);
1232
1233 if (ret_freq && policy->cur &&
1234 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1235 /* verify no discrepancy between actual and
1236 saved value exists */
1237 if (unlikely(ret_freq != policy->cur)) {
1238 cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1239 schedule_work(&policy->update);
1240 }
1241 }
1242
1243 return ret_freq;
1244}
1245
1246/**
1247 * cpufreq_get - get the current CPU frequency (in kHz)
1248 * @cpu: CPU number
1249 *
1250 * Get the CPU current (static) CPU frequency
1251 */
1252unsigned int cpufreq_get(unsigned int cpu)
1253{
1254 unsigned int ret_freq = 0;
1255 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1256
1257 if (!policy)
1258 goto out;
1259
1260 if (unlikely(lock_policy_rwsem_read(cpu)))
1261 goto out_policy;
1262
1263 ret_freq = __cpufreq_get(cpu);
1264
1265 unlock_policy_rwsem_read(cpu);
1266
1267out_policy:
1268 cpufreq_cpu_put(policy);
1269out:
1270 return ret_freq;
1271}
1272EXPORT_SYMBOL(cpufreq_get);
1273
1274static struct sysdev_driver cpufreq_sysdev_driver = {
1275 .add = cpufreq_add_dev,
1276 .remove = cpufreq_remove_dev,
1277};
1278
1279
1280/**
1281 * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1282 *
1283 * This function is only executed for the boot processor. The other CPUs
1284 * have been put offline by means of CPU hotplug.
1285 */
1286static int cpufreq_bp_suspend(void)
1287{
1288 int ret = 0;
1289
1290 int cpu = smp_processor_id();
1291 struct cpufreq_policy *cpu_policy;
1292
1293 pr_debug("suspending cpu %u\n", cpu);
1294
1295 /* If there's no policy for the boot CPU, we have nothing to do. */
1296 cpu_policy = cpufreq_cpu_get(cpu);
1297 if (!cpu_policy)
1298 return 0;
1299
1300 if (cpufreq_driver->suspend) {
1301 ret = cpufreq_driver->suspend(cpu_policy);
1302 if (ret)
1303 printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1304 "step on CPU %u\n", cpu_policy->cpu);
1305 }
1306
1307 cpufreq_cpu_put(cpu_policy);
1308 return ret;
1309}
1310
1311/**
1312 * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1313 *
1314 * 1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1315 * 2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1316 * restored. It will verify that the current freq is in sync with
1317 * what we believe it to be. This is a bit later than when it
1318 * should be, but nonethteless it's better than calling
1319 * cpufreq_driver->get() here which might re-enable interrupts...
1320 *
1321 * This function is only executed for the boot CPU. The other CPUs have not
1322 * been turned on yet.
1323 */
1324static void cpufreq_bp_resume(void)
1325{
1326 int ret = 0;
1327
1328 int cpu = smp_processor_id();
1329 struct cpufreq_policy *cpu_policy;
1330
1331 pr_debug("resuming cpu %u\n", cpu);
1332
1333 /* If there's no policy for the boot CPU, we have nothing to do. */
1334 cpu_policy = cpufreq_cpu_get(cpu);
1335 if (!cpu_policy)
1336 return;
1337
1338 if (cpufreq_driver->resume) {
1339 ret = cpufreq_driver->resume(cpu_policy);
1340 if (ret) {
1341 printk(KERN_ERR "cpufreq: resume failed in ->resume "
1342 "step on CPU %u\n", cpu_policy->cpu);
1343 goto fail;
1344 }
1345 }
1346
1347 schedule_work(&cpu_policy->update);
1348
1349fail:
1350 cpufreq_cpu_put(cpu_policy);
1351}
1352
1353static struct syscore_ops cpufreq_syscore_ops = {
1354 .suspend = cpufreq_bp_suspend,
1355 .resume = cpufreq_bp_resume,
1356};
1357
1358
1359/*********************************************************************
1360 * NOTIFIER LISTS INTERFACE *
1361 *********************************************************************/
1362
1363/**
1364 * cpufreq_register_notifier - register a driver with cpufreq
1365 * @nb: notifier function to register
1366 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1367 *
1368 * Add a driver to one of two lists: either a list of drivers that
1369 * are notified about clock rate changes (once before and once after
1370 * the transition), or a list of drivers that are notified about
1371 * changes in cpufreq policy.
1372 *
1373 * This function may sleep, and has the same return conditions as
1374 * blocking_notifier_chain_register.
1375 */
1376int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1377{
1378 int ret;
1379
1380 WARN_ON(!init_cpufreq_transition_notifier_list_called);
1381
1382 switch (list) {
1383 case CPUFREQ_TRANSITION_NOTIFIER:
1384 ret = srcu_notifier_chain_register(
1385 &cpufreq_transition_notifier_list, nb);
1386 break;
1387 case CPUFREQ_POLICY_NOTIFIER:
1388 ret = blocking_notifier_chain_register(
1389 &cpufreq_policy_notifier_list, nb);
1390 break;
1391 default:
1392 ret = -EINVAL;
1393 }
1394
1395 return ret;
1396}
1397EXPORT_SYMBOL(cpufreq_register_notifier);
1398
1399
1400/**
1401 * cpufreq_unregister_notifier - unregister a driver with cpufreq
1402 * @nb: notifier block to be unregistered
1403 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1404 *
1405 * Remove a driver from the CPU frequency notifier list.
1406 *
1407 * This function may sleep, and has the same return conditions as
1408 * blocking_notifier_chain_unregister.
1409 */
1410int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1411{
1412 int ret;
1413
1414 switch (list) {
1415 case CPUFREQ_TRANSITION_NOTIFIER:
1416 ret = srcu_notifier_chain_unregister(
1417 &cpufreq_transition_notifier_list, nb);
1418 break;
1419 case CPUFREQ_POLICY_NOTIFIER:
1420 ret = blocking_notifier_chain_unregister(
1421 &cpufreq_policy_notifier_list, nb);
1422 break;
1423 default:
1424 ret = -EINVAL;
1425 }
1426
1427 return ret;
1428}
1429EXPORT_SYMBOL(cpufreq_unregister_notifier);
1430
1431
1432/*********************************************************************
1433 * GOVERNORS *
1434 *********************************************************************/
1435
1436
1437int __cpufreq_driver_target(struct cpufreq_policy *policy,
1438 unsigned int target_freq,
1439 unsigned int relation)
1440{
1441 int retval = -EINVAL;
1442
1443 pr_debug("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1444 target_freq, relation);
1445 if (cpu_online(policy->cpu) && cpufreq_driver->target)
1446 retval = cpufreq_driver->target(policy, target_freq, relation);
1447
1448 return retval;
1449}
1450EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1451
1452int cpufreq_driver_target(struct cpufreq_policy *policy,
1453 unsigned int target_freq,
1454 unsigned int relation)
1455{
1456 int ret = -EINVAL;
1457
1458 policy = cpufreq_cpu_get(policy->cpu);
1459 if (!policy)
1460 goto no_policy;
1461
1462 if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1463 goto fail;
1464
1465 ret = __cpufreq_driver_target(policy, target_freq, relation);
1466
1467 unlock_policy_rwsem_write(policy->cpu);
1468
1469fail:
1470 cpufreq_cpu_put(policy);
1471no_policy:
1472 return ret;
1473}
1474EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1475
1476int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1477{
1478 int ret = 0;
1479
1480 policy = cpufreq_cpu_get(policy->cpu);
1481 if (!policy)
1482 return -EINVAL;
1483
1484 if (cpu_online(cpu) && cpufreq_driver->getavg)
1485 ret = cpufreq_driver->getavg(policy, cpu);
1486
1487 cpufreq_cpu_put(policy);
1488 return ret;
1489}
1490EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1491
1492/*
1493 * when "event" is CPUFREQ_GOV_LIMITS
1494 */
1495
1496static int __cpufreq_governor(struct cpufreq_policy *policy,
1497 unsigned int event)
1498{
1499 int ret;
1500
1501 /* Only must be defined when default governor is known to have latency
1502 restrictions, like e.g. conservative or ondemand.
1503 That this is the case is already ensured in Kconfig
1504 */
1505#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1506 struct cpufreq_governor *gov = &cpufreq_gov_performance;
1507#else
1508 struct cpufreq_governor *gov = NULL;
1509#endif
1510
1511 if (policy->governor->max_transition_latency &&
1512 policy->cpuinfo.transition_latency >
1513 policy->governor->max_transition_latency) {
1514 if (!gov)
1515 return -EINVAL;
1516 else {
1517 printk(KERN_WARNING "%s governor failed, too long"
1518 " transition latency of HW, fallback"
1519 " to %s governor\n",
1520 policy->governor->name,
1521 gov->name);
1522 policy->governor = gov;
1523 }
1524 }
1525
1526 if (!try_module_get(policy->governor->owner))
1527 return -EINVAL;
1528
1529 pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1530 policy->cpu, event);
1531 ret = policy->governor->governor(policy, event);
1532
1533 /* we keep one module reference alive for
1534 each CPU governed by this CPU */
1535 if ((event != CPUFREQ_GOV_START) || ret)
1536 module_put(policy->governor->owner);
1537 if ((event == CPUFREQ_GOV_STOP) && !ret)
1538 module_put(policy->governor->owner);
1539
1540 return ret;
1541}
1542
1543
1544int cpufreq_register_governor(struct cpufreq_governor *governor)
1545{
1546 int err;
1547
1548 if (!governor)
1549 return -EINVAL;
1550
1551 mutex_lock(&cpufreq_governor_mutex);
1552
1553 err = -EBUSY;
1554 if (__find_governor(governor->name) == NULL) {
1555 err = 0;
1556 list_add(&governor->governor_list, &cpufreq_governor_list);
1557 }
1558
1559 mutex_unlock(&cpufreq_governor_mutex);
1560 return err;
1561}
1562EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1563
1564
1565void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1566{
1567#ifdef CONFIG_HOTPLUG_CPU
1568 int cpu;
1569#endif
1570
1571 if (!governor)
1572 return;
1573
1574#ifdef CONFIG_HOTPLUG_CPU
1575 for_each_present_cpu(cpu) {
1576 if (cpu_online(cpu))
1577 continue;
1578 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1579 strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1580 }
1581#endif
1582
1583 mutex_lock(&cpufreq_governor_mutex);
1584 list_del(&governor->governor_list);
1585 mutex_unlock(&cpufreq_governor_mutex);
1586 return;
1587}
1588EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1589
1590
1591
1592/*********************************************************************
1593 * POLICY INTERFACE *
1594 *********************************************************************/
1595
1596/**
1597 * cpufreq_get_policy - get the current cpufreq_policy
1598 * @policy: struct cpufreq_policy into which the current cpufreq_policy
1599 * is written
1600 *
1601 * Reads the current cpufreq policy.
1602 */
1603int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1604{
1605 struct cpufreq_policy *cpu_policy;
1606 if (!policy)
1607 return -EINVAL;
1608
1609 cpu_policy = cpufreq_cpu_get(cpu);
1610 if (!cpu_policy)
1611 return -EINVAL;
1612
1613 memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1614
1615 cpufreq_cpu_put(cpu_policy);
1616 return 0;
1617}
1618EXPORT_SYMBOL(cpufreq_get_policy);
1619
1620
1621/*
1622 * data : current policy.
1623 * policy : policy to be set.
1624 */
1625static int __cpufreq_set_policy(struct cpufreq_policy *data,
1626 struct cpufreq_policy *policy)
1627{
1628 int ret = 0;
1629
1630 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1631 policy->min, policy->max);
1632
1633 memcpy(&policy->cpuinfo, &data->cpuinfo,
1634 sizeof(struct cpufreq_cpuinfo));
1635
1636 if (policy->min > data->max || policy->max < data->min) {
1637 ret = -EINVAL;
1638 goto error_out;
1639 }
1640
1641 /* verify the cpu speed can be set within this limit */
1642 ret = cpufreq_driver->verify(policy);
1643 if (ret)
1644 goto error_out;
1645
1646 /* adjust if necessary - all reasons */
1647 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1648 CPUFREQ_ADJUST, policy);
1649
1650 /* adjust if necessary - hardware incompatibility*/
1651 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1652 CPUFREQ_INCOMPATIBLE, policy);
1653
1654 /* verify the cpu speed can be set within this limit,
1655 which might be different to the first one */
1656 ret = cpufreq_driver->verify(policy);
1657 if (ret)
1658 goto error_out;
1659
1660 /* notification of the new policy */
1661 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1662 CPUFREQ_NOTIFY, policy);
1663
1664 data->min = policy->min;
1665 data->max = policy->max;
1666
1667 pr_debug("new min and max freqs are %u - %u kHz\n",
1668 data->min, data->max);
1669
1670 if (cpufreq_driver->setpolicy) {
1671 data->policy = policy->policy;
1672 pr_debug("setting range\n");
1673 ret = cpufreq_driver->setpolicy(policy);
1674 } else {
1675 if (policy->governor != data->governor) {
1676 /* save old, working values */
1677 struct cpufreq_governor *old_gov = data->governor;
1678
1679 pr_debug("governor switch\n");
1680
1681 /* end old governor */
1682 if (data->governor)
1683 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1684
1685 /* start new governor */
1686 data->governor = policy->governor;
1687 if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1688 /* new governor failed, so re-start old one */
1689 pr_debug("starting governor %s failed\n",
1690 data->governor->name);
1691 if (old_gov) {
1692 data->governor = old_gov;
1693 __cpufreq_governor(data,
1694 CPUFREQ_GOV_START);
1695 }
1696 ret = -EINVAL;
1697 goto error_out;
1698 }
1699 /* might be a policy change, too, so fall through */
1700 }
1701 pr_debug("governor: change or update limits\n");
1702 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1703 }
1704
1705error_out:
1706 return ret;
1707}
1708
1709/**
1710 * cpufreq_update_policy - re-evaluate an existing cpufreq policy
1711 * @cpu: CPU which shall be re-evaluated
1712 *
1713 * Useful for policy notifiers which have different necessities
1714 * at different times.
1715 */
1716int cpufreq_update_policy(unsigned int cpu)
1717{
1718 struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1719 struct cpufreq_policy policy;
1720 int ret;
1721
1722 if (!data) {
1723 ret = -ENODEV;
1724 goto no_policy;
1725 }
1726
1727 if (unlikely(lock_policy_rwsem_write(cpu))) {
1728 ret = -EINVAL;
1729 goto fail;
1730 }
1731
1732 pr_debug("updating policy for CPU %u\n", cpu);
1733 memcpy(&policy, data, sizeof(struct cpufreq_policy));
1734 policy.min = data->user_policy.min;
1735 policy.max = data->user_policy.max;
1736 policy.policy = data->user_policy.policy;
1737 policy.governor = data->user_policy.governor;
1738
1739 /* BIOS might change freq behind our back
1740 -> ask driver for current freq and notify governors about a change */
1741 if (cpufreq_driver->get) {
1742 policy.cur = cpufreq_driver->get(cpu);
1743 if (!data->cur) {
1744 pr_debug("Driver did not initialize current freq");
1745 data->cur = policy.cur;
1746 } else {
1747 if (data->cur != policy.cur)
1748 cpufreq_out_of_sync(cpu, data->cur,
1749 policy.cur);
1750 }
1751 }
1752
1753 ret = __cpufreq_set_policy(data, &policy);
1754
1755 unlock_policy_rwsem_write(cpu);
1756
1757fail:
1758 cpufreq_cpu_put(data);
1759no_policy:
1760 return ret;
1761}
1762EXPORT_SYMBOL(cpufreq_update_policy);
1763
1764static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1765 unsigned long action, void *hcpu)
1766{
1767 unsigned int cpu = (unsigned long)hcpu;
1768 struct sys_device *sys_dev;
1769
1770 sys_dev = get_cpu_sysdev(cpu);
1771 if (sys_dev) {
1772 switch (action) {
1773 case CPU_ONLINE:
1774 case CPU_ONLINE_FROZEN:
1775 cpufreq_add_dev(sys_dev);
1776 break;
1777 case CPU_DOWN_PREPARE:
1778 case CPU_DOWN_PREPARE_FROZEN:
1779 if (unlikely(lock_policy_rwsem_write(cpu)))
1780 BUG();
1781
1782 __cpufreq_remove_dev(sys_dev);
1783 break;
1784 case CPU_DOWN_FAILED:
1785 case CPU_DOWN_FAILED_FROZEN:
1786 cpufreq_add_dev(sys_dev);
1787 break;
1788 }
1789 }
1790 return NOTIFY_OK;
1791}
1792
1793static struct notifier_block __refdata cpufreq_cpu_notifier = {
1794 .notifier_call = cpufreq_cpu_callback,
1795};
1796
1797/*********************************************************************
1798 * REGISTER / UNREGISTER CPUFREQ DRIVER *
1799 *********************************************************************/
1800
1801/**
1802 * cpufreq_register_driver - register a CPU Frequency driver
1803 * @driver_data: A struct cpufreq_driver containing the values#
1804 * submitted by the CPU Frequency driver.
1805 *
1806 * Registers a CPU Frequency driver to this core code. This code
1807 * returns zero on success, -EBUSY when another driver got here first
1808 * (and isn't unregistered in the meantime).
1809 *
1810 */
1811int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1812{
1813 unsigned long flags;
1814 int ret;
1815
1816 if (!driver_data || !driver_data->verify || !driver_data->init ||
1817 ((!driver_data->setpolicy) && (!driver_data->target)))
1818 return -EINVAL;
1819
1820 pr_debug("trying to register driver %s\n", driver_data->name);
1821
1822 if (driver_data->setpolicy)
1823 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1824
1825 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1826 if (cpufreq_driver) {
1827 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1828 return -EBUSY;
1829 }
1830 cpufreq_driver = driver_data;
1831 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1832
1833 ret = sysdev_driver_register(&cpu_sysdev_class,
1834 &cpufreq_sysdev_driver);
1835 if (ret)
1836 goto err_null_driver;
1837
1838 if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1839 int i;
1840 ret = -ENODEV;
1841
1842 /* check for at least one working CPU */
1843 for (i = 0; i < nr_cpu_ids; i++)
1844 if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1845 ret = 0;
1846 break;
1847 }
1848
1849 /* if all ->init() calls failed, unregister */
1850 if (ret) {
1851 pr_debug("no CPU initialized for driver %s\n",
1852 driver_data->name);
1853 goto err_sysdev_unreg;
1854 }
1855 }
1856
1857 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1858 pr_debug("driver %s up and running\n", driver_data->name);
1859
1860 return 0;
1861err_sysdev_unreg:
1862 sysdev_driver_unregister(&cpu_sysdev_class,
1863 &cpufreq_sysdev_driver);
1864err_null_driver:
1865 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1866 cpufreq_driver = NULL;
1867 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1868 return ret;
1869}
1870EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1871
1872
1873/**
1874 * cpufreq_unregister_driver - unregister the current CPUFreq driver
1875 *
1876 * Unregister the current CPUFreq driver. Only call this if you have
1877 * the right to do so, i.e. if you have succeeded in initialising before!
1878 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1879 * currently not initialised.
1880 */
1881int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1882{
1883 unsigned long flags;
1884
1885 if (!cpufreq_driver || (driver != cpufreq_driver))
1886 return -EINVAL;
1887
1888 pr_debug("unregistering driver %s\n", driver->name);
1889
1890 sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1891 unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1892
1893 spin_lock_irqsave(&cpufreq_driver_lock, flags);
1894 cpufreq_driver = NULL;
1895 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1896
1897 return 0;
1898}
1899EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1900
1901static int __init cpufreq_core_init(void)
1902{
1903 int cpu;
1904
1905 for_each_possible_cpu(cpu) {
1906 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1907 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1908 }
1909
1910 cpufreq_global_kobject = kobject_create_and_add("cpufreq",
1911 &cpu_sysdev_class.kset.kobj);
1912 BUG_ON(!cpufreq_global_kobject);
1913 register_syscore_ops(&cpufreq_syscore_ops);
1914
1915 return 0;
1916}
1917core_initcall(cpufreq_core_init);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/drivers/cpufreq/cpufreq.c
4 *
5 * Copyright (C) 2001 Russell King
6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8 *
9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10 * Added handling for CPU hotplug
11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12 * Fix handling for CPU hotplug -- affected CPUs
13 */
14
15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17#include <linux/cpu.h>
18#include <linux/cpufreq.h>
19#include <linux/cpu_cooling.h>
20#include <linux/delay.h>
21#include <linux/device.h>
22#include <linux/init.h>
23#include <linux/kernel_stat.h>
24#include <linux/module.h>
25#include <linux/mutex.h>
26#include <linux/pm_qos.h>
27#include <linux/slab.h>
28#include <linux/suspend.h>
29#include <linux/syscore_ops.h>
30#include <linux/tick.h>
31#include <trace/events/power.h>
32
33static LIST_HEAD(cpufreq_policy_list);
34
35/* Macros to iterate over CPU policies */
36#define for_each_suitable_policy(__policy, __active) \
37 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
38 if ((__active) == !policy_is_inactive(__policy))
39
40#define for_each_active_policy(__policy) \
41 for_each_suitable_policy(__policy, true)
42#define for_each_inactive_policy(__policy) \
43 for_each_suitable_policy(__policy, false)
44
45/* Iterate over governors */
46static LIST_HEAD(cpufreq_governor_list);
47#define for_each_governor(__governor) \
48 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
49
50static char default_governor[CPUFREQ_NAME_LEN];
51
52/*
53 * The "cpufreq driver" - the arch- or hardware-dependent low
54 * level driver of CPUFreq support, and its spinlock. This lock
55 * also protects the cpufreq_cpu_data array.
56 */
57static struct cpufreq_driver *cpufreq_driver;
58static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
59static DEFINE_RWLOCK(cpufreq_driver_lock);
60
61static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
62bool cpufreq_supports_freq_invariance(void)
63{
64 return static_branch_likely(&cpufreq_freq_invariance);
65}
66
67/* Flag to suspend/resume CPUFreq governors */
68static bool cpufreq_suspended;
69
70static inline bool has_target(void)
71{
72 return cpufreq_driver->target_index || cpufreq_driver->target;
73}
74
75/* internal prototypes */
76static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
77static int cpufreq_init_governor(struct cpufreq_policy *policy);
78static void cpufreq_exit_governor(struct cpufreq_policy *policy);
79static void cpufreq_governor_limits(struct cpufreq_policy *policy);
80static int cpufreq_set_policy(struct cpufreq_policy *policy,
81 struct cpufreq_governor *new_gov,
82 unsigned int new_pol);
83
84/*
85 * Two notifier lists: the "policy" list is involved in the
86 * validation process for a new CPU frequency policy; the
87 * "transition" list for kernel code that needs to handle
88 * changes to devices when the CPU clock speed changes.
89 * The mutex locks both lists.
90 */
91static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
92SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
93
94static int off __read_mostly;
95static int cpufreq_disabled(void)
96{
97 return off;
98}
99void disable_cpufreq(void)
100{
101 off = 1;
102}
103static DEFINE_MUTEX(cpufreq_governor_mutex);
104
105bool have_governor_per_policy(void)
106{
107 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
108}
109EXPORT_SYMBOL_GPL(have_governor_per_policy);
110
111static struct kobject *cpufreq_global_kobject;
112
113struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
114{
115 if (have_governor_per_policy())
116 return &policy->kobj;
117 else
118 return cpufreq_global_kobject;
119}
120EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
121
122static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
123{
124 struct kernel_cpustat kcpustat;
125 u64 cur_wall_time;
126 u64 idle_time;
127 u64 busy_time;
128
129 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
130
131 kcpustat_cpu_fetch(&kcpustat, cpu);
132
133 busy_time = kcpustat.cpustat[CPUTIME_USER];
134 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
135 busy_time += kcpustat.cpustat[CPUTIME_IRQ];
136 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
137 busy_time += kcpustat.cpustat[CPUTIME_STEAL];
138 busy_time += kcpustat.cpustat[CPUTIME_NICE];
139
140 idle_time = cur_wall_time - busy_time;
141 if (wall)
142 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
143
144 return div_u64(idle_time, NSEC_PER_USEC);
145}
146
147u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
148{
149 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
150
151 if (idle_time == -1ULL)
152 return get_cpu_idle_time_jiffy(cpu, wall);
153 else if (!io_busy)
154 idle_time += get_cpu_iowait_time_us(cpu, wall);
155
156 return idle_time;
157}
158EXPORT_SYMBOL_GPL(get_cpu_idle_time);
159
160/*
161 * This is a generic cpufreq init() routine which can be used by cpufreq
162 * drivers of SMP systems. It will do following:
163 * - validate & show freq table passed
164 * - set policies transition latency
165 * - policy->cpus with all possible CPUs
166 */
167void cpufreq_generic_init(struct cpufreq_policy *policy,
168 struct cpufreq_frequency_table *table,
169 unsigned int transition_latency)
170{
171 policy->freq_table = table;
172 policy->cpuinfo.transition_latency = transition_latency;
173
174 /*
175 * The driver only supports the SMP configuration where all processors
176 * share the clock and voltage and clock.
177 */
178 cpumask_setall(policy->cpus);
179}
180EXPORT_SYMBOL_GPL(cpufreq_generic_init);
181
182struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
183{
184 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
185
186 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
187}
188EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
189
190unsigned int cpufreq_generic_get(unsigned int cpu)
191{
192 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
193
194 if (!policy || IS_ERR(policy->clk)) {
195 pr_err("%s: No %s associated to cpu: %d\n",
196 __func__, policy ? "clk" : "policy", cpu);
197 return 0;
198 }
199
200 return clk_get_rate(policy->clk) / 1000;
201}
202EXPORT_SYMBOL_GPL(cpufreq_generic_get);
203
204/**
205 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
206 * @cpu: CPU to find the policy for.
207 *
208 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
209 * the kobject reference counter of that policy. Return a valid policy on
210 * success or NULL on failure.
211 *
212 * The policy returned by this function has to be released with the help of
213 * cpufreq_cpu_put() to balance its kobject reference counter properly.
214 */
215struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
216{
217 struct cpufreq_policy *policy = NULL;
218 unsigned long flags;
219
220 if (WARN_ON(cpu >= nr_cpu_ids))
221 return NULL;
222
223 /* get the cpufreq driver */
224 read_lock_irqsave(&cpufreq_driver_lock, flags);
225
226 if (cpufreq_driver) {
227 /* get the CPU */
228 policy = cpufreq_cpu_get_raw(cpu);
229 if (policy)
230 kobject_get(&policy->kobj);
231 }
232
233 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
234
235 return policy;
236}
237EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
238
239/**
240 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
241 * @policy: cpufreq policy returned by cpufreq_cpu_get().
242 */
243void cpufreq_cpu_put(struct cpufreq_policy *policy)
244{
245 kobject_put(&policy->kobj);
246}
247EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
248
249/**
250 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
251 * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
252 */
253void cpufreq_cpu_release(struct cpufreq_policy *policy)
254{
255 if (WARN_ON(!policy))
256 return;
257
258 lockdep_assert_held(&policy->rwsem);
259
260 up_write(&policy->rwsem);
261
262 cpufreq_cpu_put(policy);
263}
264
265/**
266 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
267 * @cpu: CPU to find the policy for.
268 *
269 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
270 * if the policy returned by it is not NULL, acquire its rwsem for writing.
271 * Return the policy if it is active or release it and return NULL otherwise.
272 *
273 * The policy returned by this function has to be released with the help of
274 * cpufreq_cpu_release() in order to release its rwsem and balance its usage
275 * counter properly.
276 */
277struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
278{
279 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
280
281 if (!policy)
282 return NULL;
283
284 down_write(&policy->rwsem);
285
286 if (policy_is_inactive(policy)) {
287 cpufreq_cpu_release(policy);
288 return NULL;
289 }
290
291 return policy;
292}
293
294/*********************************************************************
295 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
296 *********************************************************************/
297
298/**
299 * adjust_jiffies - Adjust the system "loops_per_jiffy".
300 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
301 * @ci: Frequency change information.
302 *
303 * This function alters the system "loops_per_jiffy" for the clock
304 * speed change. Note that loops_per_jiffy cannot be updated on SMP
305 * systems as each CPU might be scaled differently. So, use the arch
306 * per-CPU loops_per_jiffy value wherever possible.
307 */
308static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
309{
310#ifndef CONFIG_SMP
311 static unsigned long l_p_j_ref;
312 static unsigned int l_p_j_ref_freq;
313
314 if (ci->flags & CPUFREQ_CONST_LOOPS)
315 return;
316
317 if (!l_p_j_ref_freq) {
318 l_p_j_ref = loops_per_jiffy;
319 l_p_j_ref_freq = ci->old;
320 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
321 l_p_j_ref, l_p_j_ref_freq);
322 }
323 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
324 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
325 ci->new);
326 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
327 loops_per_jiffy, ci->new);
328 }
329#endif
330}
331
332/**
333 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
334 * @policy: cpufreq policy to enable fast frequency switching for.
335 * @freqs: contain details of the frequency update.
336 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
337 *
338 * This function calls the transition notifiers and adjust_jiffies().
339 *
340 * It is called twice on all CPU frequency changes that have external effects.
341 */
342static void cpufreq_notify_transition(struct cpufreq_policy *policy,
343 struct cpufreq_freqs *freqs,
344 unsigned int state)
345{
346 int cpu;
347
348 BUG_ON(irqs_disabled());
349
350 if (cpufreq_disabled())
351 return;
352
353 freqs->policy = policy;
354 freqs->flags = cpufreq_driver->flags;
355 pr_debug("notification %u of frequency transition to %u kHz\n",
356 state, freqs->new);
357
358 switch (state) {
359 case CPUFREQ_PRECHANGE:
360 /*
361 * Detect if the driver reported a value as "old frequency"
362 * which is not equal to what the cpufreq core thinks is
363 * "old frequency".
364 */
365 if (policy->cur && policy->cur != freqs->old) {
366 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
367 freqs->old, policy->cur);
368 freqs->old = policy->cur;
369 }
370
371 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
372 CPUFREQ_PRECHANGE, freqs);
373
374 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
375 break;
376
377 case CPUFREQ_POSTCHANGE:
378 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
379 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
380 cpumask_pr_args(policy->cpus));
381
382 for_each_cpu(cpu, policy->cpus)
383 trace_cpu_frequency(freqs->new, cpu);
384
385 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
386 CPUFREQ_POSTCHANGE, freqs);
387
388 cpufreq_stats_record_transition(policy, freqs->new);
389 policy->cur = freqs->new;
390 }
391}
392
393/* Do post notifications when there are chances that transition has failed */
394static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
395 struct cpufreq_freqs *freqs, int transition_failed)
396{
397 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
398 if (!transition_failed)
399 return;
400
401 swap(freqs->old, freqs->new);
402 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
403 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
404}
405
406void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
407 struct cpufreq_freqs *freqs)
408{
409
410 /*
411 * Catch double invocations of _begin() which lead to self-deadlock.
412 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
413 * doesn't invoke _begin() on their behalf, and hence the chances of
414 * double invocations are very low. Moreover, there are scenarios
415 * where these checks can emit false-positive warnings in these
416 * drivers; so we avoid that by skipping them altogether.
417 */
418 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
419 && current == policy->transition_task);
420
421wait:
422 wait_event(policy->transition_wait, !policy->transition_ongoing);
423
424 spin_lock(&policy->transition_lock);
425
426 if (unlikely(policy->transition_ongoing)) {
427 spin_unlock(&policy->transition_lock);
428 goto wait;
429 }
430
431 policy->transition_ongoing = true;
432 policy->transition_task = current;
433
434 spin_unlock(&policy->transition_lock);
435
436 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
437}
438EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
439
440void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
441 struct cpufreq_freqs *freqs, int transition_failed)
442{
443 if (WARN_ON(!policy->transition_ongoing))
444 return;
445
446 cpufreq_notify_post_transition(policy, freqs, transition_failed);
447
448 arch_set_freq_scale(policy->related_cpus,
449 policy->cur,
450 policy->cpuinfo.max_freq);
451
452 policy->transition_ongoing = false;
453 policy->transition_task = NULL;
454
455 wake_up(&policy->transition_wait);
456}
457EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
458
459/*
460 * Fast frequency switching status count. Positive means "enabled", negative
461 * means "disabled" and 0 means "not decided yet".
462 */
463static int cpufreq_fast_switch_count;
464static DEFINE_MUTEX(cpufreq_fast_switch_lock);
465
466static void cpufreq_list_transition_notifiers(void)
467{
468 struct notifier_block *nb;
469
470 pr_info("Registered transition notifiers:\n");
471
472 mutex_lock(&cpufreq_transition_notifier_list.mutex);
473
474 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
475 pr_info("%pS\n", nb->notifier_call);
476
477 mutex_unlock(&cpufreq_transition_notifier_list.mutex);
478}
479
480/**
481 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
482 * @policy: cpufreq policy to enable fast frequency switching for.
483 *
484 * Try to enable fast frequency switching for @policy.
485 *
486 * The attempt will fail if there is at least one transition notifier registered
487 * at this point, as fast frequency switching is quite fundamentally at odds
488 * with transition notifiers. Thus if successful, it will make registration of
489 * transition notifiers fail going forward.
490 */
491void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
492{
493 lockdep_assert_held(&policy->rwsem);
494
495 if (!policy->fast_switch_possible)
496 return;
497
498 mutex_lock(&cpufreq_fast_switch_lock);
499 if (cpufreq_fast_switch_count >= 0) {
500 cpufreq_fast_switch_count++;
501 policy->fast_switch_enabled = true;
502 } else {
503 pr_warn("CPU%u: Fast frequency switching not enabled\n",
504 policy->cpu);
505 cpufreq_list_transition_notifiers();
506 }
507 mutex_unlock(&cpufreq_fast_switch_lock);
508}
509EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
510
511/**
512 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
513 * @policy: cpufreq policy to disable fast frequency switching for.
514 */
515void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
516{
517 mutex_lock(&cpufreq_fast_switch_lock);
518 if (policy->fast_switch_enabled) {
519 policy->fast_switch_enabled = false;
520 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
521 cpufreq_fast_switch_count--;
522 }
523 mutex_unlock(&cpufreq_fast_switch_lock);
524}
525EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
526
527static unsigned int __resolve_freq(struct cpufreq_policy *policy,
528 unsigned int target_freq, unsigned int relation)
529{
530 unsigned int idx;
531
532 target_freq = clamp_val(target_freq, policy->min, policy->max);
533
534 if (!cpufreq_driver->target_index)
535 return target_freq;
536
537 idx = cpufreq_frequency_table_target(policy, target_freq, relation);
538 policy->cached_resolved_idx = idx;
539 policy->cached_target_freq = target_freq;
540 return policy->freq_table[idx].frequency;
541}
542
543/**
544 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
545 * one.
546 * @policy: associated policy to interrogate
547 * @target_freq: target frequency to resolve.
548 *
549 * The target to driver frequency mapping is cached in the policy.
550 *
551 * Return: Lowest driver-supported frequency greater than or equal to the
552 * given target_freq, subject to policy (min/max) and driver limitations.
553 */
554unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
555 unsigned int target_freq)
556{
557 return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_L);
558}
559EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
560
561unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
562{
563 unsigned int latency;
564
565 if (policy->transition_delay_us)
566 return policy->transition_delay_us;
567
568 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
569 if (latency) {
570 /*
571 * For platforms that can change the frequency very fast (< 10
572 * us), the above formula gives a decent transition delay. But
573 * for platforms where transition_latency is in milliseconds, it
574 * ends up giving unrealistic values.
575 *
576 * Cap the default transition delay to 10 ms, which seems to be
577 * a reasonable amount of time after which we should reevaluate
578 * the frequency.
579 */
580 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
581 }
582
583 return LATENCY_MULTIPLIER;
584}
585EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
586
587/*********************************************************************
588 * SYSFS INTERFACE *
589 *********************************************************************/
590static ssize_t show_boost(struct kobject *kobj,
591 struct kobj_attribute *attr, char *buf)
592{
593 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
594}
595
596static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
597 const char *buf, size_t count)
598{
599 int ret, enable;
600
601 ret = sscanf(buf, "%d", &enable);
602 if (ret != 1 || enable < 0 || enable > 1)
603 return -EINVAL;
604
605 if (cpufreq_boost_trigger_state(enable)) {
606 pr_err("%s: Cannot %s BOOST!\n",
607 __func__, enable ? "enable" : "disable");
608 return -EINVAL;
609 }
610
611 pr_debug("%s: cpufreq BOOST %s\n",
612 __func__, enable ? "enabled" : "disabled");
613
614 return count;
615}
616define_one_global_rw(boost);
617
618static struct cpufreq_governor *find_governor(const char *str_governor)
619{
620 struct cpufreq_governor *t;
621
622 for_each_governor(t)
623 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
624 return t;
625
626 return NULL;
627}
628
629static struct cpufreq_governor *get_governor(const char *str_governor)
630{
631 struct cpufreq_governor *t;
632
633 mutex_lock(&cpufreq_governor_mutex);
634 t = find_governor(str_governor);
635 if (!t)
636 goto unlock;
637
638 if (!try_module_get(t->owner))
639 t = NULL;
640
641unlock:
642 mutex_unlock(&cpufreq_governor_mutex);
643
644 return t;
645}
646
647static unsigned int cpufreq_parse_policy(char *str_governor)
648{
649 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
650 return CPUFREQ_POLICY_PERFORMANCE;
651
652 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
653 return CPUFREQ_POLICY_POWERSAVE;
654
655 return CPUFREQ_POLICY_UNKNOWN;
656}
657
658/**
659 * cpufreq_parse_governor - parse a governor string only for has_target()
660 * @str_governor: Governor name.
661 */
662static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
663{
664 struct cpufreq_governor *t;
665
666 t = get_governor(str_governor);
667 if (t)
668 return t;
669
670 if (request_module("cpufreq_%s", str_governor))
671 return NULL;
672
673 return get_governor(str_governor);
674}
675
676/*
677 * cpufreq_per_cpu_attr_read() / show_##file_name() -
678 * print out cpufreq information
679 *
680 * Write out information from cpufreq_driver->policy[cpu]; object must be
681 * "unsigned int".
682 */
683
684#define show_one(file_name, object) \
685static ssize_t show_##file_name \
686(struct cpufreq_policy *policy, char *buf) \
687{ \
688 return sprintf(buf, "%u\n", policy->object); \
689}
690
691show_one(cpuinfo_min_freq, cpuinfo.min_freq);
692show_one(cpuinfo_max_freq, cpuinfo.max_freq);
693show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
694show_one(scaling_min_freq, min);
695show_one(scaling_max_freq, max);
696
697__weak unsigned int arch_freq_get_on_cpu(int cpu)
698{
699 return 0;
700}
701
702static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
703{
704 ssize_t ret;
705 unsigned int freq;
706
707 freq = arch_freq_get_on_cpu(policy->cpu);
708 if (freq)
709 ret = sprintf(buf, "%u\n", freq);
710 else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
711 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
712 else
713 ret = sprintf(buf, "%u\n", policy->cur);
714 return ret;
715}
716
717/*
718 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
719 */
720#define store_one(file_name, object) \
721static ssize_t store_##file_name \
722(struct cpufreq_policy *policy, const char *buf, size_t count) \
723{ \
724 unsigned long val; \
725 int ret; \
726 \
727 ret = sscanf(buf, "%lu", &val); \
728 if (ret != 1) \
729 return -EINVAL; \
730 \
731 ret = freq_qos_update_request(policy->object##_freq_req, val);\
732 return ret >= 0 ? count : ret; \
733}
734
735store_one(scaling_min_freq, min);
736store_one(scaling_max_freq, max);
737
738/*
739 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
740 */
741static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
742 char *buf)
743{
744 unsigned int cur_freq = __cpufreq_get(policy);
745
746 if (cur_freq)
747 return sprintf(buf, "%u\n", cur_freq);
748
749 return sprintf(buf, "<unknown>\n");
750}
751
752/*
753 * show_scaling_governor - show the current policy for the specified CPU
754 */
755static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
756{
757 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
758 return sprintf(buf, "powersave\n");
759 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
760 return sprintf(buf, "performance\n");
761 else if (policy->governor)
762 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
763 policy->governor->name);
764 return -EINVAL;
765}
766
767/*
768 * store_scaling_governor - store policy for the specified CPU
769 */
770static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
771 const char *buf, size_t count)
772{
773 char str_governor[16];
774 int ret;
775
776 ret = sscanf(buf, "%15s", str_governor);
777 if (ret != 1)
778 return -EINVAL;
779
780 if (cpufreq_driver->setpolicy) {
781 unsigned int new_pol;
782
783 new_pol = cpufreq_parse_policy(str_governor);
784 if (!new_pol)
785 return -EINVAL;
786
787 ret = cpufreq_set_policy(policy, NULL, new_pol);
788 } else {
789 struct cpufreq_governor *new_gov;
790
791 new_gov = cpufreq_parse_governor(str_governor);
792 if (!new_gov)
793 return -EINVAL;
794
795 ret = cpufreq_set_policy(policy, new_gov,
796 CPUFREQ_POLICY_UNKNOWN);
797
798 module_put(new_gov->owner);
799 }
800
801 return ret ? ret : count;
802}
803
804/*
805 * show_scaling_driver - show the cpufreq driver currently loaded
806 */
807static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
808{
809 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
810}
811
812/*
813 * show_scaling_available_governors - show the available CPUfreq governors
814 */
815static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
816 char *buf)
817{
818 ssize_t i = 0;
819 struct cpufreq_governor *t;
820
821 if (!has_target()) {
822 i += sprintf(buf, "performance powersave");
823 goto out;
824 }
825
826 mutex_lock(&cpufreq_governor_mutex);
827 for_each_governor(t) {
828 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
829 - (CPUFREQ_NAME_LEN + 2)))
830 break;
831 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
832 }
833 mutex_unlock(&cpufreq_governor_mutex);
834out:
835 i += sprintf(&buf[i], "\n");
836 return i;
837}
838
839ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
840{
841 ssize_t i = 0;
842 unsigned int cpu;
843
844 for_each_cpu(cpu, mask) {
845 if (i)
846 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
847 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
848 if (i >= (PAGE_SIZE - 5))
849 break;
850 }
851 i += sprintf(&buf[i], "\n");
852 return i;
853}
854EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
855
856/*
857 * show_related_cpus - show the CPUs affected by each transition even if
858 * hw coordination is in use
859 */
860static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
861{
862 return cpufreq_show_cpus(policy->related_cpus, buf);
863}
864
865/*
866 * show_affected_cpus - show the CPUs affected by each transition
867 */
868static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
869{
870 return cpufreq_show_cpus(policy->cpus, buf);
871}
872
873static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
874 const char *buf, size_t count)
875{
876 unsigned int freq = 0;
877 unsigned int ret;
878
879 if (!policy->governor || !policy->governor->store_setspeed)
880 return -EINVAL;
881
882 ret = sscanf(buf, "%u", &freq);
883 if (ret != 1)
884 return -EINVAL;
885
886 policy->governor->store_setspeed(policy, freq);
887
888 return count;
889}
890
891static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
892{
893 if (!policy->governor || !policy->governor->show_setspeed)
894 return sprintf(buf, "<unsupported>\n");
895
896 return policy->governor->show_setspeed(policy, buf);
897}
898
899/*
900 * show_bios_limit - show the current cpufreq HW/BIOS limitation
901 */
902static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
903{
904 unsigned int limit;
905 int ret;
906 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
907 if (!ret)
908 return sprintf(buf, "%u\n", limit);
909 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
910}
911
912cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
913cpufreq_freq_attr_ro(cpuinfo_min_freq);
914cpufreq_freq_attr_ro(cpuinfo_max_freq);
915cpufreq_freq_attr_ro(cpuinfo_transition_latency);
916cpufreq_freq_attr_ro(scaling_available_governors);
917cpufreq_freq_attr_ro(scaling_driver);
918cpufreq_freq_attr_ro(scaling_cur_freq);
919cpufreq_freq_attr_ro(bios_limit);
920cpufreq_freq_attr_ro(related_cpus);
921cpufreq_freq_attr_ro(affected_cpus);
922cpufreq_freq_attr_rw(scaling_min_freq);
923cpufreq_freq_attr_rw(scaling_max_freq);
924cpufreq_freq_attr_rw(scaling_governor);
925cpufreq_freq_attr_rw(scaling_setspeed);
926
927static struct attribute *default_attrs[] = {
928 &cpuinfo_min_freq.attr,
929 &cpuinfo_max_freq.attr,
930 &cpuinfo_transition_latency.attr,
931 &scaling_min_freq.attr,
932 &scaling_max_freq.attr,
933 &affected_cpus.attr,
934 &related_cpus.attr,
935 &scaling_governor.attr,
936 &scaling_driver.attr,
937 &scaling_available_governors.attr,
938 &scaling_setspeed.attr,
939 NULL
940};
941
942#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
943#define to_attr(a) container_of(a, struct freq_attr, attr)
944
945static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
946{
947 struct cpufreq_policy *policy = to_policy(kobj);
948 struct freq_attr *fattr = to_attr(attr);
949 ssize_t ret;
950
951 if (!fattr->show)
952 return -EIO;
953
954 down_read(&policy->rwsem);
955 ret = fattr->show(policy, buf);
956 up_read(&policy->rwsem);
957
958 return ret;
959}
960
961static ssize_t store(struct kobject *kobj, struct attribute *attr,
962 const char *buf, size_t count)
963{
964 struct cpufreq_policy *policy = to_policy(kobj);
965 struct freq_attr *fattr = to_attr(attr);
966 ssize_t ret = -EINVAL;
967
968 if (!fattr->store)
969 return -EIO;
970
971 /*
972 * cpus_read_trylock() is used here to work around a circular lock
973 * dependency problem with respect to the cpufreq_register_driver().
974 */
975 if (!cpus_read_trylock())
976 return -EBUSY;
977
978 if (cpu_online(policy->cpu)) {
979 down_write(&policy->rwsem);
980 ret = fattr->store(policy, buf, count);
981 up_write(&policy->rwsem);
982 }
983
984 cpus_read_unlock();
985
986 return ret;
987}
988
989static void cpufreq_sysfs_release(struct kobject *kobj)
990{
991 struct cpufreq_policy *policy = to_policy(kobj);
992 pr_debug("last reference is dropped\n");
993 complete(&policy->kobj_unregister);
994}
995
996static const struct sysfs_ops sysfs_ops = {
997 .show = show,
998 .store = store,
999};
1000
1001static struct kobj_type ktype_cpufreq = {
1002 .sysfs_ops = &sysfs_ops,
1003 .default_attrs = default_attrs,
1004 .release = cpufreq_sysfs_release,
1005};
1006
1007static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
1008{
1009 struct device *dev = get_cpu_device(cpu);
1010
1011 if (unlikely(!dev))
1012 return;
1013
1014 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1015 return;
1016
1017 dev_dbg(dev, "%s: Adding symlink\n", __func__);
1018 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1019 dev_err(dev, "cpufreq symlink creation failed\n");
1020}
1021
1022static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
1023 struct device *dev)
1024{
1025 dev_dbg(dev, "%s: Removing symlink\n", __func__);
1026 sysfs_remove_link(&dev->kobj, "cpufreq");
1027}
1028
1029static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1030{
1031 struct freq_attr **drv_attr;
1032 int ret = 0;
1033
1034 /* set up files for this cpu device */
1035 drv_attr = cpufreq_driver->attr;
1036 while (drv_attr && *drv_attr) {
1037 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1038 if (ret)
1039 return ret;
1040 drv_attr++;
1041 }
1042 if (cpufreq_driver->get) {
1043 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1044 if (ret)
1045 return ret;
1046 }
1047
1048 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1049 if (ret)
1050 return ret;
1051
1052 if (cpufreq_driver->bios_limit) {
1053 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1054 if (ret)
1055 return ret;
1056 }
1057
1058 return 0;
1059}
1060
1061static int cpufreq_init_policy(struct cpufreq_policy *policy)
1062{
1063 struct cpufreq_governor *gov = NULL;
1064 unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1065 int ret;
1066
1067 if (has_target()) {
1068 /* Update policy governor to the one used before hotplug. */
1069 gov = get_governor(policy->last_governor);
1070 if (gov) {
1071 pr_debug("Restoring governor %s for cpu %d\n",
1072 gov->name, policy->cpu);
1073 } else {
1074 gov = get_governor(default_governor);
1075 }
1076
1077 if (!gov) {
1078 gov = cpufreq_default_governor();
1079 __module_get(gov->owner);
1080 }
1081
1082 } else {
1083
1084 /* Use the default policy if there is no last_policy. */
1085 if (policy->last_policy) {
1086 pol = policy->last_policy;
1087 } else {
1088 pol = cpufreq_parse_policy(default_governor);
1089 /*
1090 * In case the default governor is neither "performance"
1091 * nor "powersave", fall back to the initial policy
1092 * value set by the driver.
1093 */
1094 if (pol == CPUFREQ_POLICY_UNKNOWN)
1095 pol = policy->policy;
1096 }
1097 if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1098 pol != CPUFREQ_POLICY_POWERSAVE)
1099 return -ENODATA;
1100 }
1101
1102 ret = cpufreq_set_policy(policy, gov, pol);
1103 if (gov)
1104 module_put(gov->owner);
1105
1106 return ret;
1107}
1108
1109static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1110{
1111 int ret = 0;
1112
1113 /* Has this CPU been taken care of already? */
1114 if (cpumask_test_cpu(cpu, policy->cpus))
1115 return 0;
1116
1117 down_write(&policy->rwsem);
1118 if (has_target())
1119 cpufreq_stop_governor(policy);
1120
1121 cpumask_set_cpu(cpu, policy->cpus);
1122
1123 if (has_target()) {
1124 ret = cpufreq_start_governor(policy);
1125 if (ret)
1126 pr_err("%s: Failed to start governor\n", __func__);
1127 }
1128 up_write(&policy->rwsem);
1129 return ret;
1130}
1131
1132void refresh_frequency_limits(struct cpufreq_policy *policy)
1133{
1134 if (!policy_is_inactive(policy)) {
1135 pr_debug("updating policy for CPU %u\n", policy->cpu);
1136
1137 cpufreq_set_policy(policy, policy->governor, policy->policy);
1138 }
1139}
1140EXPORT_SYMBOL(refresh_frequency_limits);
1141
1142static void handle_update(struct work_struct *work)
1143{
1144 struct cpufreq_policy *policy =
1145 container_of(work, struct cpufreq_policy, update);
1146
1147 pr_debug("handle_update for cpu %u called\n", policy->cpu);
1148 down_write(&policy->rwsem);
1149 refresh_frequency_limits(policy);
1150 up_write(&policy->rwsem);
1151}
1152
1153static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1154 void *data)
1155{
1156 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1157
1158 schedule_work(&policy->update);
1159 return 0;
1160}
1161
1162static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1163 void *data)
1164{
1165 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1166
1167 schedule_work(&policy->update);
1168 return 0;
1169}
1170
1171static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1172{
1173 struct kobject *kobj;
1174 struct completion *cmp;
1175
1176 down_write(&policy->rwsem);
1177 cpufreq_stats_free_table(policy);
1178 kobj = &policy->kobj;
1179 cmp = &policy->kobj_unregister;
1180 up_write(&policy->rwsem);
1181 kobject_put(kobj);
1182
1183 /*
1184 * We need to make sure that the underlying kobj is
1185 * actually not referenced anymore by anybody before we
1186 * proceed with unloading.
1187 */
1188 pr_debug("waiting for dropping of refcount\n");
1189 wait_for_completion(cmp);
1190 pr_debug("wait complete\n");
1191}
1192
1193static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1194{
1195 struct cpufreq_policy *policy;
1196 struct device *dev = get_cpu_device(cpu);
1197 int ret;
1198
1199 if (!dev)
1200 return NULL;
1201
1202 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1203 if (!policy)
1204 return NULL;
1205
1206 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1207 goto err_free_policy;
1208
1209 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1210 goto err_free_cpumask;
1211
1212 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1213 goto err_free_rcpumask;
1214
1215 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1216 cpufreq_global_kobject, "policy%u", cpu);
1217 if (ret) {
1218 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1219 /*
1220 * The entire policy object will be freed below, but the extra
1221 * memory allocated for the kobject name needs to be freed by
1222 * releasing the kobject.
1223 */
1224 kobject_put(&policy->kobj);
1225 goto err_free_real_cpus;
1226 }
1227
1228 freq_constraints_init(&policy->constraints);
1229
1230 policy->nb_min.notifier_call = cpufreq_notifier_min;
1231 policy->nb_max.notifier_call = cpufreq_notifier_max;
1232
1233 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1234 &policy->nb_min);
1235 if (ret) {
1236 dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n",
1237 ret, cpumask_pr_args(policy->cpus));
1238 goto err_kobj_remove;
1239 }
1240
1241 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1242 &policy->nb_max);
1243 if (ret) {
1244 dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n",
1245 ret, cpumask_pr_args(policy->cpus));
1246 goto err_min_qos_notifier;
1247 }
1248
1249 INIT_LIST_HEAD(&policy->policy_list);
1250 init_rwsem(&policy->rwsem);
1251 spin_lock_init(&policy->transition_lock);
1252 init_waitqueue_head(&policy->transition_wait);
1253 init_completion(&policy->kobj_unregister);
1254 INIT_WORK(&policy->update, handle_update);
1255
1256 policy->cpu = cpu;
1257 return policy;
1258
1259err_min_qos_notifier:
1260 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1261 &policy->nb_min);
1262err_kobj_remove:
1263 cpufreq_policy_put_kobj(policy);
1264err_free_real_cpus:
1265 free_cpumask_var(policy->real_cpus);
1266err_free_rcpumask:
1267 free_cpumask_var(policy->related_cpus);
1268err_free_cpumask:
1269 free_cpumask_var(policy->cpus);
1270err_free_policy:
1271 kfree(policy);
1272
1273 return NULL;
1274}
1275
1276static void cpufreq_policy_free(struct cpufreq_policy *policy)
1277{
1278 unsigned long flags;
1279 int cpu;
1280
1281 /* Remove policy from list */
1282 write_lock_irqsave(&cpufreq_driver_lock, flags);
1283 list_del(&policy->policy_list);
1284
1285 for_each_cpu(cpu, policy->related_cpus)
1286 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1287 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1288
1289 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1290 &policy->nb_max);
1291 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1292 &policy->nb_min);
1293
1294 /* Cancel any pending policy->update work before freeing the policy. */
1295 cancel_work_sync(&policy->update);
1296
1297 if (policy->max_freq_req) {
1298 /*
1299 * CPUFREQ_CREATE_POLICY notification is sent only after
1300 * successfully adding max_freq_req request.
1301 */
1302 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1303 CPUFREQ_REMOVE_POLICY, policy);
1304 freq_qos_remove_request(policy->max_freq_req);
1305 }
1306
1307 freq_qos_remove_request(policy->min_freq_req);
1308 kfree(policy->min_freq_req);
1309
1310 cpufreq_policy_put_kobj(policy);
1311 free_cpumask_var(policy->real_cpus);
1312 free_cpumask_var(policy->related_cpus);
1313 free_cpumask_var(policy->cpus);
1314 kfree(policy);
1315}
1316
1317static int cpufreq_online(unsigned int cpu)
1318{
1319 struct cpufreq_policy *policy;
1320 bool new_policy;
1321 unsigned long flags;
1322 unsigned int j;
1323 int ret;
1324
1325 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1326
1327 /* Check if this CPU already has a policy to manage it */
1328 policy = per_cpu(cpufreq_cpu_data, cpu);
1329 if (policy) {
1330 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1331 if (!policy_is_inactive(policy))
1332 return cpufreq_add_policy_cpu(policy, cpu);
1333
1334 /* This is the only online CPU for the policy. Start over. */
1335 new_policy = false;
1336 down_write(&policy->rwsem);
1337 policy->cpu = cpu;
1338 policy->governor = NULL;
1339 up_write(&policy->rwsem);
1340 } else {
1341 new_policy = true;
1342 policy = cpufreq_policy_alloc(cpu);
1343 if (!policy)
1344 return -ENOMEM;
1345 }
1346
1347 if (!new_policy && cpufreq_driver->online) {
1348 ret = cpufreq_driver->online(policy);
1349 if (ret) {
1350 pr_debug("%s: %d: initialization failed\n", __func__,
1351 __LINE__);
1352 goto out_exit_policy;
1353 }
1354
1355 /* Recover policy->cpus using related_cpus */
1356 cpumask_copy(policy->cpus, policy->related_cpus);
1357 } else {
1358 cpumask_copy(policy->cpus, cpumask_of(cpu));
1359
1360 /*
1361 * Call driver. From then on the cpufreq must be able
1362 * to accept all calls to ->verify and ->setpolicy for this CPU.
1363 */
1364 ret = cpufreq_driver->init(policy);
1365 if (ret) {
1366 pr_debug("%s: %d: initialization failed\n", __func__,
1367 __LINE__);
1368 goto out_free_policy;
1369 }
1370
1371 /*
1372 * The initialization has succeeded and the policy is online.
1373 * If there is a problem with its frequency table, take it
1374 * offline and drop it.
1375 */
1376 ret = cpufreq_table_validate_and_sort(policy);
1377 if (ret)
1378 goto out_offline_policy;
1379
1380 /* related_cpus should at least include policy->cpus. */
1381 cpumask_copy(policy->related_cpus, policy->cpus);
1382 }
1383
1384 down_write(&policy->rwsem);
1385 /*
1386 * affected cpus must always be the one, which are online. We aren't
1387 * managing offline cpus here.
1388 */
1389 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1390
1391 if (new_policy) {
1392 for_each_cpu(j, policy->related_cpus) {
1393 per_cpu(cpufreq_cpu_data, j) = policy;
1394 add_cpu_dev_symlink(policy, j);
1395 }
1396
1397 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1398 GFP_KERNEL);
1399 if (!policy->min_freq_req) {
1400 ret = -ENOMEM;
1401 goto out_destroy_policy;
1402 }
1403
1404 ret = freq_qos_add_request(&policy->constraints,
1405 policy->min_freq_req, FREQ_QOS_MIN,
1406 policy->min);
1407 if (ret < 0) {
1408 /*
1409 * So we don't call freq_qos_remove_request() for an
1410 * uninitialized request.
1411 */
1412 kfree(policy->min_freq_req);
1413 policy->min_freq_req = NULL;
1414 goto out_destroy_policy;
1415 }
1416
1417 /*
1418 * This must be initialized right here to avoid calling
1419 * freq_qos_remove_request() on uninitialized request in case
1420 * of errors.
1421 */
1422 policy->max_freq_req = policy->min_freq_req + 1;
1423
1424 ret = freq_qos_add_request(&policy->constraints,
1425 policy->max_freq_req, FREQ_QOS_MAX,
1426 policy->max);
1427 if (ret < 0) {
1428 policy->max_freq_req = NULL;
1429 goto out_destroy_policy;
1430 }
1431
1432 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1433 CPUFREQ_CREATE_POLICY, policy);
1434 }
1435
1436 if (cpufreq_driver->get && has_target()) {
1437 policy->cur = cpufreq_driver->get(policy->cpu);
1438 if (!policy->cur) {
1439 ret = -EIO;
1440 pr_err("%s: ->get() failed\n", __func__);
1441 goto out_destroy_policy;
1442 }
1443 }
1444
1445 /*
1446 * Sometimes boot loaders set CPU frequency to a value outside of
1447 * frequency table present with cpufreq core. In such cases CPU might be
1448 * unstable if it has to run on that frequency for long duration of time
1449 * and so its better to set it to a frequency which is specified in
1450 * freq-table. This also makes cpufreq stats inconsistent as
1451 * cpufreq-stats would fail to register because current frequency of CPU
1452 * isn't found in freq-table.
1453 *
1454 * Because we don't want this change to effect boot process badly, we go
1455 * for the next freq which is >= policy->cur ('cur' must be set by now,
1456 * otherwise we will end up setting freq to lowest of the table as 'cur'
1457 * is initialized to zero).
1458 *
1459 * We are passing target-freq as "policy->cur - 1" otherwise
1460 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1461 * equal to target-freq.
1462 */
1463 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1464 && has_target()) {
1465 unsigned int old_freq = policy->cur;
1466
1467 /* Are we running at unknown frequency ? */
1468 ret = cpufreq_frequency_table_get_index(policy, old_freq);
1469 if (ret == -EINVAL) {
1470 ret = __cpufreq_driver_target(policy, old_freq - 1,
1471 CPUFREQ_RELATION_L);
1472
1473 /*
1474 * Reaching here after boot in a few seconds may not
1475 * mean that system will remain stable at "unknown"
1476 * frequency for longer duration. Hence, a BUG_ON().
1477 */
1478 BUG_ON(ret);
1479 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
1480 __func__, policy->cpu, old_freq, policy->cur);
1481 }
1482 }
1483
1484 if (new_policy) {
1485 ret = cpufreq_add_dev_interface(policy);
1486 if (ret)
1487 goto out_destroy_policy;
1488
1489 cpufreq_stats_create_table(policy);
1490
1491 write_lock_irqsave(&cpufreq_driver_lock, flags);
1492 list_add(&policy->policy_list, &cpufreq_policy_list);
1493 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1494 }
1495
1496 ret = cpufreq_init_policy(policy);
1497 if (ret) {
1498 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1499 __func__, cpu, ret);
1500 goto out_destroy_policy;
1501 }
1502
1503 up_write(&policy->rwsem);
1504
1505 kobject_uevent(&policy->kobj, KOBJ_ADD);
1506
1507 /* Callback for handling stuff after policy is ready */
1508 if (cpufreq_driver->ready)
1509 cpufreq_driver->ready(policy);
1510
1511 if (cpufreq_thermal_control_enabled(cpufreq_driver))
1512 policy->cdev = of_cpufreq_cooling_register(policy);
1513
1514 pr_debug("initialization complete\n");
1515
1516 return 0;
1517
1518out_destroy_policy:
1519 for_each_cpu(j, policy->real_cpus)
1520 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1521
1522 up_write(&policy->rwsem);
1523
1524out_offline_policy:
1525 if (cpufreq_driver->offline)
1526 cpufreq_driver->offline(policy);
1527
1528out_exit_policy:
1529 if (cpufreq_driver->exit)
1530 cpufreq_driver->exit(policy);
1531
1532out_free_policy:
1533 cpufreq_policy_free(policy);
1534 return ret;
1535}
1536
1537/**
1538 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1539 * @dev: CPU device.
1540 * @sif: Subsystem interface structure pointer (not used)
1541 */
1542static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1543{
1544 struct cpufreq_policy *policy;
1545 unsigned cpu = dev->id;
1546 int ret;
1547
1548 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1549
1550 if (cpu_online(cpu)) {
1551 ret = cpufreq_online(cpu);
1552 if (ret)
1553 return ret;
1554 }
1555
1556 /* Create sysfs link on CPU registration */
1557 policy = per_cpu(cpufreq_cpu_data, cpu);
1558 if (policy)
1559 add_cpu_dev_symlink(policy, cpu);
1560
1561 return 0;
1562}
1563
1564static int cpufreq_offline(unsigned int cpu)
1565{
1566 struct cpufreq_policy *policy;
1567 int ret;
1568
1569 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1570
1571 policy = cpufreq_cpu_get_raw(cpu);
1572 if (!policy) {
1573 pr_debug("%s: No cpu_data found\n", __func__);
1574 return 0;
1575 }
1576
1577 down_write(&policy->rwsem);
1578 if (has_target())
1579 cpufreq_stop_governor(policy);
1580
1581 cpumask_clear_cpu(cpu, policy->cpus);
1582
1583 if (policy_is_inactive(policy)) {
1584 if (has_target())
1585 strncpy(policy->last_governor, policy->governor->name,
1586 CPUFREQ_NAME_LEN);
1587 else
1588 policy->last_policy = policy->policy;
1589 } else if (cpu == policy->cpu) {
1590 /* Nominate new CPU */
1591 policy->cpu = cpumask_any(policy->cpus);
1592 }
1593
1594 /* Start governor again for active policy */
1595 if (!policy_is_inactive(policy)) {
1596 if (has_target()) {
1597 ret = cpufreq_start_governor(policy);
1598 if (ret)
1599 pr_err("%s: Failed to start governor\n", __func__);
1600 }
1601
1602 goto unlock;
1603 }
1604
1605 if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1606 cpufreq_cooling_unregister(policy->cdev);
1607 policy->cdev = NULL;
1608 }
1609
1610 if (has_target())
1611 cpufreq_exit_governor(policy);
1612
1613 /*
1614 * Perform the ->offline() during light-weight tear-down, as
1615 * that allows fast recovery when the CPU comes back.
1616 */
1617 if (cpufreq_driver->offline) {
1618 cpufreq_driver->offline(policy);
1619 } else if (cpufreq_driver->exit) {
1620 cpufreq_driver->exit(policy);
1621 policy->freq_table = NULL;
1622 }
1623
1624unlock:
1625 up_write(&policy->rwsem);
1626 return 0;
1627}
1628
1629/*
1630 * cpufreq_remove_dev - remove a CPU device
1631 *
1632 * Removes the cpufreq interface for a CPU device.
1633 */
1634static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1635{
1636 unsigned int cpu = dev->id;
1637 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1638
1639 if (!policy)
1640 return;
1641
1642 if (cpu_online(cpu))
1643 cpufreq_offline(cpu);
1644
1645 cpumask_clear_cpu(cpu, policy->real_cpus);
1646 remove_cpu_dev_symlink(policy, dev);
1647
1648 if (cpumask_empty(policy->real_cpus)) {
1649 /* We did light-weight exit earlier, do full tear down now */
1650 if (cpufreq_driver->offline)
1651 cpufreq_driver->exit(policy);
1652
1653 cpufreq_policy_free(policy);
1654 }
1655}
1656
1657/**
1658 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1659 * @policy: Policy managing CPUs.
1660 * @new_freq: New CPU frequency.
1661 *
1662 * Adjust to the current frequency first and clean up later by either calling
1663 * cpufreq_update_policy(), or scheduling handle_update().
1664 */
1665static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1666 unsigned int new_freq)
1667{
1668 struct cpufreq_freqs freqs;
1669
1670 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1671 policy->cur, new_freq);
1672
1673 freqs.old = policy->cur;
1674 freqs.new = new_freq;
1675
1676 cpufreq_freq_transition_begin(policy, &freqs);
1677 cpufreq_freq_transition_end(policy, &freqs, 0);
1678}
1679
1680static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1681{
1682 unsigned int new_freq;
1683
1684 new_freq = cpufreq_driver->get(policy->cpu);
1685 if (!new_freq)
1686 return 0;
1687
1688 /*
1689 * If fast frequency switching is used with the given policy, the check
1690 * against policy->cur is pointless, so skip it in that case.
1691 */
1692 if (policy->fast_switch_enabled || !has_target())
1693 return new_freq;
1694
1695 if (policy->cur != new_freq) {
1696 cpufreq_out_of_sync(policy, new_freq);
1697 if (update)
1698 schedule_work(&policy->update);
1699 }
1700
1701 return new_freq;
1702}
1703
1704/**
1705 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1706 * @cpu: CPU number
1707 *
1708 * This is the last known freq, without actually getting it from the driver.
1709 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1710 */
1711unsigned int cpufreq_quick_get(unsigned int cpu)
1712{
1713 struct cpufreq_policy *policy;
1714 unsigned int ret_freq = 0;
1715 unsigned long flags;
1716
1717 read_lock_irqsave(&cpufreq_driver_lock, flags);
1718
1719 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1720 ret_freq = cpufreq_driver->get(cpu);
1721 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1722 return ret_freq;
1723 }
1724
1725 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1726
1727 policy = cpufreq_cpu_get(cpu);
1728 if (policy) {
1729 ret_freq = policy->cur;
1730 cpufreq_cpu_put(policy);
1731 }
1732
1733 return ret_freq;
1734}
1735EXPORT_SYMBOL(cpufreq_quick_get);
1736
1737/**
1738 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1739 * @cpu: CPU number
1740 *
1741 * Just return the max possible frequency for a given CPU.
1742 */
1743unsigned int cpufreq_quick_get_max(unsigned int cpu)
1744{
1745 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1746 unsigned int ret_freq = 0;
1747
1748 if (policy) {
1749 ret_freq = policy->max;
1750 cpufreq_cpu_put(policy);
1751 }
1752
1753 return ret_freq;
1754}
1755EXPORT_SYMBOL(cpufreq_quick_get_max);
1756
1757/**
1758 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1759 * @cpu: CPU number
1760 *
1761 * The default return value is the max_freq field of cpuinfo.
1762 */
1763__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1764{
1765 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1766 unsigned int ret_freq = 0;
1767
1768 if (policy) {
1769 ret_freq = policy->cpuinfo.max_freq;
1770 cpufreq_cpu_put(policy);
1771 }
1772
1773 return ret_freq;
1774}
1775EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1776
1777static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1778{
1779 if (unlikely(policy_is_inactive(policy)))
1780 return 0;
1781
1782 return cpufreq_verify_current_freq(policy, true);
1783}
1784
1785/**
1786 * cpufreq_get - get the current CPU frequency (in kHz)
1787 * @cpu: CPU number
1788 *
1789 * Get the CPU current (static) CPU frequency
1790 */
1791unsigned int cpufreq_get(unsigned int cpu)
1792{
1793 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1794 unsigned int ret_freq = 0;
1795
1796 if (policy) {
1797 down_read(&policy->rwsem);
1798 if (cpufreq_driver->get)
1799 ret_freq = __cpufreq_get(policy);
1800 up_read(&policy->rwsem);
1801
1802 cpufreq_cpu_put(policy);
1803 }
1804
1805 return ret_freq;
1806}
1807EXPORT_SYMBOL(cpufreq_get);
1808
1809static struct subsys_interface cpufreq_interface = {
1810 .name = "cpufreq",
1811 .subsys = &cpu_subsys,
1812 .add_dev = cpufreq_add_dev,
1813 .remove_dev = cpufreq_remove_dev,
1814};
1815
1816/*
1817 * In case platform wants some specific frequency to be configured
1818 * during suspend..
1819 */
1820int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1821{
1822 int ret;
1823
1824 if (!policy->suspend_freq) {
1825 pr_debug("%s: suspend_freq not defined\n", __func__);
1826 return 0;
1827 }
1828
1829 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1830 policy->suspend_freq);
1831
1832 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1833 CPUFREQ_RELATION_H);
1834 if (ret)
1835 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1836 __func__, policy->suspend_freq, ret);
1837
1838 return ret;
1839}
1840EXPORT_SYMBOL(cpufreq_generic_suspend);
1841
1842/**
1843 * cpufreq_suspend() - Suspend CPUFreq governors.
1844 *
1845 * Called during system wide Suspend/Hibernate cycles for suspending governors
1846 * as some platforms can't change frequency after this point in suspend cycle.
1847 * Because some of the devices (like: i2c, regulators, etc) they use for
1848 * changing frequency are suspended quickly after this point.
1849 */
1850void cpufreq_suspend(void)
1851{
1852 struct cpufreq_policy *policy;
1853
1854 if (!cpufreq_driver)
1855 return;
1856
1857 if (!has_target() && !cpufreq_driver->suspend)
1858 goto suspend;
1859
1860 pr_debug("%s: Suspending Governors\n", __func__);
1861
1862 for_each_active_policy(policy) {
1863 if (has_target()) {
1864 down_write(&policy->rwsem);
1865 cpufreq_stop_governor(policy);
1866 up_write(&policy->rwsem);
1867 }
1868
1869 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1870 pr_err("%s: Failed to suspend driver: %s\n", __func__,
1871 cpufreq_driver->name);
1872 }
1873
1874suspend:
1875 cpufreq_suspended = true;
1876}
1877
1878/**
1879 * cpufreq_resume() - Resume CPUFreq governors.
1880 *
1881 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1882 * are suspended with cpufreq_suspend().
1883 */
1884void cpufreq_resume(void)
1885{
1886 struct cpufreq_policy *policy;
1887 int ret;
1888
1889 if (!cpufreq_driver)
1890 return;
1891
1892 if (unlikely(!cpufreq_suspended))
1893 return;
1894
1895 cpufreq_suspended = false;
1896
1897 if (!has_target() && !cpufreq_driver->resume)
1898 return;
1899
1900 pr_debug("%s: Resuming Governors\n", __func__);
1901
1902 for_each_active_policy(policy) {
1903 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1904 pr_err("%s: Failed to resume driver: %p\n", __func__,
1905 policy);
1906 } else if (has_target()) {
1907 down_write(&policy->rwsem);
1908 ret = cpufreq_start_governor(policy);
1909 up_write(&policy->rwsem);
1910
1911 if (ret)
1912 pr_err("%s: Failed to start governor for policy: %p\n",
1913 __func__, policy);
1914 }
1915 }
1916}
1917
1918/**
1919 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
1920 * @flags: Flags to test against the current cpufreq driver's flags.
1921 *
1922 * Assumes that the driver is there, so callers must ensure that this is the
1923 * case.
1924 */
1925bool cpufreq_driver_test_flags(u16 flags)
1926{
1927 return !!(cpufreq_driver->flags & flags);
1928}
1929
1930/**
1931 * cpufreq_get_current_driver - Return the current driver's name.
1932 *
1933 * Return the name string of the currently registered cpufreq driver or NULL if
1934 * none.
1935 */
1936const char *cpufreq_get_current_driver(void)
1937{
1938 if (cpufreq_driver)
1939 return cpufreq_driver->name;
1940
1941 return NULL;
1942}
1943EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1944
1945/**
1946 * cpufreq_get_driver_data - Return current driver data.
1947 *
1948 * Return the private data of the currently registered cpufreq driver, or NULL
1949 * if no cpufreq driver has been registered.
1950 */
1951void *cpufreq_get_driver_data(void)
1952{
1953 if (cpufreq_driver)
1954 return cpufreq_driver->driver_data;
1955
1956 return NULL;
1957}
1958EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1959
1960/*********************************************************************
1961 * NOTIFIER LISTS INTERFACE *
1962 *********************************************************************/
1963
1964/**
1965 * cpufreq_register_notifier - Register a notifier with cpufreq.
1966 * @nb: notifier function to register.
1967 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
1968 *
1969 * Add a notifier to one of two lists: either a list of notifiers that run on
1970 * clock rate changes (once before and once after every transition), or a list
1971 * of notifiers that ron on cpufreq policy changes.
1972 *
1973 * This function may sleep and it has the same return values as
1974 * blocking_notifier_chain_register().
1975 */
1976int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1977{
1978 int ret;
1979
1980 if (cpufreq_disabled())
1981 return -EINVAL;
1982
1983 switch (list) {
1984 case CPUFREQ_TRANSITION_NOTIFIER:
1985 mutex_lock(&cpufreq_fast_switch_lock);
1986
1987 if (cpufreq_fast_switch_count > 0) {
1988 mutex_unlock(&cpufreq_fast_switch_lock);
1989 return -EBUSY;
1990 }
1991 ret = srcu_notifier_chain_register(
1992 &cpufreq_transition_notifier_list, nb);
1993 if (!ret)
1994 cpufreq_fast_switch_count--;
1995
1996 mutex_unlock(&cpufreq_fast_switch_lock);
1997 break;
1998 case CPUFREQ_POLICY_NOTIFIER:
1999 ret = blocking_notifier_chain_register(
2000 &cpufreq_policy_notifier_list, nb);
2001 break;
2002 default:
2003 ret = -EINVAL;
2004 }
2005
2006 return ret;
2007}
2008EXPORT_SYMBOL(cpufreq_register_notifier);
2009
2010/**
2011 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2012 * @nb: notifier block to be unregistered.
2013 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2014 *
2015 * Remove a notifier from one of the cpufreq notifier lists.
2016 *
2017 * This function may sleep and it has the same return values as
2018 * blocking_notifier_chain_unregister().
2019 */
2020int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2021{
2022 int ret;
2023
2024 if (cpufreq_disabled())
2025 return -EINVAL;
2026
2027 switch (list) {
2028 case CPUFREQ_TRANSITION_NOTIFIER:
2029 mutex_lock(&cpufreq_fast_switch_lock);
2030
2031 ret = srcu_notifier_chain_unregister(
2032 &cpufreq_transition_notifier_list, nb);
2033 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2034 cpufreq_fast_switch_count++;
2035
2036 mutex_unlock(&cpufreq_fast_switch_lock);
2037 break;
2038 case CPUFREQ_POLICY_NOTIFIER:
2039 ret = blocking_notifier_chain_unregister(
2040 &cpufreq_policy_notifier_list, nb);
2041 break;
2042 default:
2043 ret = -EINVAL;
2044 }
2045
2046 return ret;
2047}
2048EXPORT_SYMBOL(cpufreq_unregister_notifier);
2049
2050
2051/*********************************************************************
2052 * GOVERNORS *
2053 *********************************************************************/
2054
2055/**
2056 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2057 * @policy: cpufreq policy to switch the frequency for.
2058 * @target_freq: New frequency to set (may be approximate).
2059 *
2060 * Carry out a fast frequency switch without sleeping.
2061 *
2062 * The driver's ->fast_switch() callback invoked by this function must be
2063 * suitable for being called from within RCU-sched read-side critical sections
2064 * and it is expected to select the minimum available frequency greater than or
2065 * equal to @target_freq (CPUFREQ_RELATION_L).
2066 *
2067 * This function must not be called if policy->fast_switch_enabled is unset.
2068 *
2069 * Governors calling this function must guarantee that it will never be invoked
2070 * twice in parallel for the same policy and that it will never be called in
2071 * parallel with either ->target() or ->target_index() for the same policy.
2072 *
2073 * Returns the actual frequency set for the CPU.
2074 *
2075 * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2076 * error condition, the hardware configuration must be preserved.
2077 */
2078unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2079 unsigned int target_freq)
2080{
2081 unsigned int freq;
2082 int cpu;
2083
2084 target_freq = clamp_val(target_freq, policy->min, policy->max);
2085 freq = cpufreq_driver->fast_switch(policy, target_freq);
2086
2087 if (!freq)
2088 return 0;
2089
2090 policy->cur = freq;
2091 arch_set_freq_scale(policy->related_cpus, freq,
2092 policy->cpuinfo.max_freq);
2093 cpufreq_stats_record_transition(policy, freq);
2094
2095 if (trace_cpu_frequency_enabled()) {
2096 for_each_cpu(cpu, policy->cpus)
2097 trace_cpu_frequency(freq, cpu);
2098 }
2099
2100 return freq;
2101}
2102EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2103
2104/**
2105 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2106 * @cpu: Target CPU.
2107 * @min_perf: Minimum (required) performance level (units of @capacity).
2108 * @target_perf: Target (desired) performance level (units of @capacity).
2109 * @capacity: Capacity of the target CPU.
2110 *
2111 * Carry out a fast performance level switch of @cpu without sleeping.
2112 *
2113 * The driver's ->adjust_perf() callback invoked by this function must be
2114 * suitable for being called from within RCU-sched read-side critical sections
2115 * and it is expected to select a suitable performance level equal to or above
2116 * @min_perf and preferably equal to or below @target_perf.
2117 *
2118 * This function must not be called if policy->fast_switch_enabled is unset.
2119 *
2120 * Governors calling this function must guarantee that it will never be invoked
2121 * twice in parallel for the same CPU and that it will never be called in
2122 * parallel with either ->target() or ->target_index() or ->fast_switch() for
2123 * the same CPU.
2124 */
2125void cpufreq_driver_adjust_perf(unsigned int cpu,
2126 unsigned long min_perf,
2127 unsigned long target_perf,
2128 unsigned long capacity)
2129{
2130 cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
2131}
2132
2133/**
2134 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2135 *
2136 * Return 'true' if the ->adjust_perf callback is present for the
2137 * current driver or 'false' otherwise.
2138 */
2139bool cpufreq_driver_has_adjust_perf(void)
2140{
2141 return !!cpufreq_driver->adjust_perf;
2142}
2143
2144/* Must set freqs->new to intermediate frequency */
2145static int __target_intermediate(struct cpufreq_policy *policy,
2146 struct cpufreq_freqs *freqs, int index)
2147{
2148 int ret;
2149
2150 freqs->new = cpufreq_driver->get_intermediate(policy, index);
2151
2152 /* We don't need to switch to intermediate freq */
2153 if (!freqs->new)
2154 return 0;
2155
2156 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2157 __func__, policy->cpu, freqs->old, freqs->new);
2158
2159 cpufreq_freq_transition_begin(policy, freqs);
2160 ret = cpufreq_driver->target_intermediate(policy, index);
2161 cpufreq_freq_transition_end(policy, freqs, ret);
2162
2163 if (ret)
2164 pr_err("%s: Failed to change to intermediate frequency: %d\n",
2165 __func__, ret);
2166
2167 return ret;
2168}
2169
2170static int __target_index(struct cpufreq_policy *policy, int index)
2171{
2172 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2173 unsigned int restore_freq, intermediate_freq = 0;
2174 unsigned int newfreq = policy->freq_table[index].frequency;
2175 int retval = -EINVAL;
2176 bool notify;
2177
2178 if (newfreq == policy->cur)
2179 return 0;
2180
2181 /* Save last value to restore later on errors */
2182 restore_freq = policy->cur;
2183
2184 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2185 if (notify) {
2186 /* Handle switching to intermediate frequency */
2187 if (cpufreq_driver->get_intermediate) {
2188 retval = __target_intermediate(policy, &freqs, index);
2189 if (retval)
2190 return retval;
2191
2192 intermediate_freq = freqs.new;
2193 /* Set old freq to intermediate */
2194 if (intermediate_freq)
2195 freqs.old = freqs.new;
2196 }
2197
2198 freqs.new = newfreq;
2199 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2200 __func__, policy->cpu, freqs.old, freqs.new);
2201
2202 cpufreq_freq_transition_begin(policy, &freqs);
2203 }
2204
2205 retval = cpufreq_driver->target_index(policy, index);
2206 if (retval)
2207 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2208 retval);
2209
2210 if (notify) {
2211 cpufreq_freq_transition_end(policy, &freqs, retval);
2212
2213 /*
2214 * Failed after setting to intermediate freq? Driver should have
2215 * reverted back to initial frequency and so should we. Check
2216 * here for intermediate_freq instead of get_intermediate, in
2217 * case we haven't switched to intermediate freq at all.
2218 */
2219 if (unlikely(retval && intermediate_freq)) {
2220 freqs.old = intermediate_freq;
2221 freqs.new = restore_freq;
2222 cpufreq_freq_transition_begin(policy, &freqs);
2223 cpufreq_freq_transition_end(policy, &freqs, 0);
2224 }
2225 }
2226
2227 return retval;
2228}
2229
2230int __cpufreq_driver_target(struct cpufreq_policy *policy,
2231 unsigned int target_freq,
2232 unsigned int relation)
2233{
2234 unsigned int old_target_freq = target_freq;
2235
2236 if (cpufreq_disabled())
2237 return -ENODEV;
2238
2239 target_freq = __resolve_freq(policy, target_freq, relation);
2240
2241 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
2242 policy->cpu, target_freq, relation, old_target_freq);
2243
2244 /*
2245 * This might look like a redundant call as we are checking it again
2246 * after finding index. But it is left intentionally for cases where
2247 * exactly same freq is called again and so we can save on few function
2248 * calls.
2249 */
2250 if (target_freq == policy->cur &&
2251 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
2252 return 0;
2253
2254 if (cpufreq_driver->target)
2255 return cpufreq_driver->target(policy, target_freq, relation);
2256
2257 if (!cpufreq_driver->target_index)
2258 return -EINVAL;
2259
2260 return __target_index(policy, policy->cached_resolved_idx);
2261}
2262EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2263
2264int cpufreq_driver_target(struct cpufreq_policy *policy,
2265 unsigned int target_freq,
2266 unsigned int relation)
2267{
2268 int ret;
2269
2270 down_write(&policy->rwsem);
2271
2272 ret = __cpufreq_driver_target(policy, target_freq, relation);
2273
2274 up_write(&policy->rwsem);
2275
2276 return ret;
2277}
2278EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2279
2280__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2281{
2282 return NULL;
2283}
2284
2285static int cpufreq_init_governor(struct cpufreq_policy *policy)
2286{
2287 int ret;
2288
2289 /* Don't start any governor operations if we are entering suspend */
2290 if (cpufreq_suspended)
2291 return 0;
2292 /*
2293 * Governor might not be initiated here if ACPI _PPC changed
2294 * notification happened, so check it.
2295 */
2296 if (!policy->governor)
2297 return -EINVAL;
2298
2299 /* Platform doesn't want dynamic frequency switching ? */
2300 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2301 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2302 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2303
2304 if (gov) {
2305 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2306 policy->governor->name, gov->name);
2307 policy->governor = gov;
2308 } else {
2309 return -EINVAL;
2310 }
2311 }
2312
2313 if (!try_module_get(policy->governor->owner))
2314 return -EINVAL;
2315
2316 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2317
2318 if (policy->governor->init) {
2319 ret = policy->governor->init(policy);
2320 if (ret) {
2321 module_put(policy->governor->owner);
2322 return ret;
2323 }
2324 }
2325
2326 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2327
2328 return 0;
2329}
2330
2331static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2332{
2333 if (cpufreq_suspended || !policy->governor)
2334 return;
2335
2336 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2337
2338 if (policy->governor->exit)
2339 policy->governor->exit(policy);
2340
2341 module_put(policy->governor->owner);
2342}
2343
2344int cpufreq_start_governor(struct cpufreq_policy *policy)
2345{
2346 int ret;
2347
2348 if (cpufreq_suspended)
2349 return 0;
2350
2351 if (!policy->governor)
2352 return -EINVAL;
2353
2354 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2355
2356 if (cpufreq_driver->get)
2357 cpufreq_verify_current_freq(policy, false);
2358
2359 if (policy->governor->start) {
2360 ret = policy->governor->start(policy);
2361 if (ret)
2362 return ret;
2363 }
2364
2365 if (policy->governor->limits)
2366 policy->governor->limits(policy);
2367
2368 return 0;
2369}
2370
2371void cpufreq_stop_governor(struct cpufreq_policy *policy)
2372{
2373 if (cpufreq_suspended || !policy->governor)
2374 return;
2375
2376 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2377
2378 if (policy->governor->stop)
2379 policy->governor->stop(policy);
2380}
2381
2382static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2383{
2384 if (cpufreq_suspended || !policy->governor)
2385 return;
2386
2387 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2388
2389 if (policy->governor->limits)
2390 policy->governor->limits(policy);
2391}
2392
2393int cpufreq_register_governor(struct cpufreq_governor *governor)
2394{
2395 int err;
2396
2397 if (!governor)
2398 return -EINVAL;
2399
2400 if (cpufreq_disabled())
2401 return -ENODEV;
2402
2403 mutex_lock(&cpufreq_governor_mutex);
2404
2405 err = -EBUSY;
2406 if (!find_governor(governor->name)) {
2407 err = 0;
2408 list_add(&governor->governor_list, &cpufreq_governor_list);
2409 }
2410
2411 mutex_unlock(&cpufreq_governor_mutex);
2412 return err;
2413}
2414EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2415
2416void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2417{
2418 struct cpufreq_policy *policy;
2419 unsigned long flags;
2420
2421 if (!governor)
2422 return;
2423
2424 if (cpufreq_disabled())
2425 return;
2426
2427 /* clear last_governor for all inactive policies */
2428 read_lock_irqsave(&cpufreq_driver_lock, flags);
2429 for_each_inactive_policy(policy) {
2430 if (!strcmp(policy->last_governor, governor->name)) {
2431 policy->governor = NULL;
2432 strcpy(policy->last_governor, "\0");
2433 }
2434 }
2435 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2436
2437 mutex_lock(&cpufreq_governor_mutex);
2438 list_del(&governor->governor_list);
2439 mutex_unlock(&cpufreq_governor_mutex);
2440}
2441EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2442
2443
2444/*********************************************************************
2445 * POLICY INTERFACE *
2446 *********************************************************************/
2447
2448/**
2449 * cpufreq_get_policy - get the current cpufreq_policy
2450 * @policy: struct cpufreq_policy into which the current cpufreq_policy
2451 * is written
2452 * @cpu: CPU to find the policy for
2453 *
2454 * Reads the current cpufreq policy.
2455 */
2456int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2457{
2458 struct cpufreq_policy *cpu_policy;
2459 if (!policy)
2460 return -EINVAL;
2461
2462 cpu_policy = cpufreq_cpu_get(cpu);
2463 if (!cpu_policy)
2464 return -EINVAL;
2465
2466 memcpy(policy, cpu_policy, sizeof(*policy));
2467
2468 cpufreq_cpu_put(cpu_policy);
2469 return 0;
2470}
2471EXPORT_SYMBOL(cpufreq_get_policy);
2472
2473/**
2474 * cpufreq_set_policy - Modify cpufreq policy parameters.
2475 * @policy: Policy object to modify.
2476 * @new_gov: Policy governor pointer.
2477 * @new_pol: Policy value (for drivers with built-in governors).
2478 *
2479 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2480 * limits to be set for the policy, update @policy with the verified limits
2481 * values and either invoke the driver's ->setpolicy() callback (if present) or
2482 * carry out a governor update for @policy. That is, run the current governor's
2483 * ->limits() callback (if @new_gov points to the same object as the one in
2484 * @policy) or replace the governor for @policy with @new_gov.
2485 *
2486 * The cpuinfo part of @policy is not updated by this function.
2487 */
2488static int cpufreq_set_policy(struct cpufreq_policy *policy,
2489 struct cpufreq_governor *new_gov,
2490 unsigned int new_pol)
2491{
2492 struct cpufreq_policy_data new_data;
2493 struct cpufreq_governor *old_gov;
2494 int ret;
2495
2496 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2497 new_data.freq_table = policy->freq_table;
2498 new_data.cpu = policy->cpu;
2499 /*
2500 * PM QoS framework collects all the requests from users and provide us
2501 * the final aggregated value here.
2502 */
2503 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2504 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2505
2506 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2507 new_data.cpu, new_data.min, new_data.max);
2508
2509 /*
2510 * Verify that the CPU speed can be set within these limits and make sure
2511 * that min <= max.
2512 */
2513 ret = cpufreq_driver->verify(&new_data);
2514 if (ret)
2515 return ret;
2516
2517 policy->min = new_data.min;
2518 policy->max = new_data.max;
2519 trace_cpu_frequency_limits(policy);
2520
2521 policy->cached_target_freq = UINT_MAX;
2522
2523 pr_debug("new min and max freqs are %u - %u kHz\n",
2524 policy->min, policy->max);
2525
2526 if (cpufreq_driver->setpolicy) {
2527 policy->policy = new_pol;
2528 pr_debug("setting range\n");
2529 return cpufreq_driver->setpolicy(policy);
2530 }
2531
2532 if (new_gov == policy->governor) {
2533 pr_debug("governor limits update\n");
2534 cpufreq_governor_limits(policy);
2535 return 0;
2536 }
2537
2538 pr_debug("governor switch\n");
2539
2540 /* save old, working values */
2541 old_gov = policy->governor;
2542 /* end old governor */
2543 if (old_gov) {
2544 cpufreq_stop_governor(policy);
2545 cpufreq_exit_governor(policy);
2546 }
2547
2548 /* start new governor */
2549 policy->governor = new_gov;
2550 ret = cpufreq_init_governor(policy);
2551 if (!ret) {
2552 ret = cpufreq_start_governor(policy);
2553 if (!ret) {
2554 pr_debug("governor change\n");
2555 sched_cpufreq_governor_change(policy, old_gov);
2556 return 0;
2557 }
2558 cpufreq_exit_governor(policy);
2559 }
2560
2561 /* new governor failed, so re-start old one */
2562 pr_debug("starting governor %s failed\n", policy->governor->name);
2563 if (old_gov) {
2564 policy->governor = old_gov;
2565 if (cpufreq_init_governor(policy))
2566 policy->governor = NULL;
2567 else
2568 cpufreq_start_governor(policy);
2569 }
2570
2571 return ret;
2572}
2573
2574/**
2575 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2576 * @cpu: CPU to re-evaluate the policy for.
2577 *
2578 * Update the current frequency for the cpufreq policy of @cpu and use
2579 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2580 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2581 * for the policy in question, among other things.
2582 */
2583void cpufreq_update_policy(unsigned int cpu)
2584{
2585 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
2586
2587 if (!policy)
2588 return;
2589
2590 /*
2591 * BIOS might change freq behind our back
2592 * -> ask driver for current freq and notify governors about a change
2593 */
2594 if (cpufreq_driver->get && has_target() &&
2595 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2596 goto unlock;
2597
2598 refresh_frequency_limits(policy);
2599
2600unlock:
2601 cpufreq_cpu_release(policy);
2602}
2603EXPORT_SYMBOL(cpufreq_update_policy);
2604
2605/**
2606 * cpufreq_update_limits - Update policy limits for a given CPU.
2607 * @cpu: CPU to update the policy limits for.
2608 *
2609 * Invoke the driver's ->update_limits callback if present or call
2610 * cpufreq_update_policy() for @cpu.
2611 */
2612void cpufreq_update_limits(unsigned int cpu)
2613{
2614 if (cpufreq_driver->update_limits)
2615 cpufreq_driver->update_limits(cpu);
2616 else
2617 cpufreq_update_policy(cpu);
2618}
2619EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2620
2621/*********************************************************************
2622 * BOOST *
2623 *********************************************************************/
2624static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2625{
2626 int ret;
2627
2628 if (!policy->freq_table)
2629 return -ENXIO;
2630
2631 ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
2632 if (ret) {
2633 pr_err("%s: Policy frequency update failed\n", __func__);
2634 return ret;
2635 }
2636
2637 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
2638 if (ret < 0)
2639 return ret;
2640
2641 return 0;
2642}
2643
2644int cpufreq_boost_trigger_state(int state)
2645{
2646 struct cpufreq_policy *policy;
2647 unsigned long flags;
2648 int ret = 0;
2649
2650 if (cpufreq_driver->boost_enabled == state)
2651 return 0;
2652
2653 write_lock_irqsave(&cpufreq_driver_lock, flags);
2654 cpufreq_driver->boost_enabled = state;
2655 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2656
2657 get_online_cpus();
2658 for_each_active_policy(policy) {
2659 ret = cpufreq_driver->set_boost(policy, state);
2660 if (ret)
2661 goto err_reset_state;
2662 }
2663 put_online_cpus();
2664
2665 return 0;
2666
2667err_reset_state:
2668 put_online_cpus();
2669
2670 write_lock_irqsave(&cpufreq_driver_lock, flags);
2671 cpufreq_driver->boost_enabled = !state;
2672 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2673
2674 pr_err("%s: Cannot %s BOOST\n",
2675 __func__, state ? "enable" : "disable");
2676
2677 return ret;
2678}
2679
2680static bool cpufreq_boost_supported(void)
2681{
2682 return cpufreq_driver->set_boost;
2683}
2684
2685static int create_boost_sysfs_file(void)
2686{
2687 int ret;
2688
2689 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2690 if (ret)
2691 pr_err("%s: cannot register global BOOST sysfs file\n",
2692 __func__);
2693
2694 return ret;
2695}
2696
2697static void remove_boost_sysfs_file(void)
2698{
2699 if (cpufreq_boost_supported())
2700 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2701}
2702
2703int cpufreq_enable_boost_support(void)
2704{
2705 if (!cpufreq_driver)
2706 return -EINVAL;
2707
2708 if (cpufreq_boost_supported())
2709 return 0;
2710
2711 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2712
2713 /* This will get removed on driver unregister */
2714 return create_boost_sysfs_file();
2715}
2716EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2717
2718int cpufreq_boost_enabled(void)
2719{
2720 return cpufreq_driver->boost_enabled;
2721}
2722EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2723
2724/*********************************************************************
2725 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2726 *********************************************************************/
2727static enum cpuhp_state hp_online;
2728
2729static int cpuhp_cpufreq_online(unsigned int cpu)
2730{
2731 cpufreq_online(cpu);
2732
2733 return 0;
2734}
2735
2736static int cpuhp_cpufreq_offline(unsigned int cpu)
2737{
2738 cpufreq_offline(cpu);
2739
2740 return 0;
2741}
2742
2743/**
2744 * cpufreq_register_driver - register a CPU Frequency driver
2745 * @driver_data: A struct cpufreq_driver containing the values#
2746 * submitted by the CPU Frequency driver.
2747 *
2748 * Registers a CPU Frequency driver to this core code. This code
2749 * returns zero on success, -EEXIST when another driver got here first
2750 * (and isn't unregistered in the meantime).
2751 *
2752 */
2753int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2754{
2755 unsigned long flags;
2756 int ret;
2757
2758 if (cpufreq_disabled())
2759 return -ENODEV;
2760
2761 /*
2762 * The cpufreq core depends heavily on the availability of device
2763 * structure, make sure they are available before proceeding further.
2764 */
2765 if (!get_cpu_device(0))
2766 return -EPROBE_DEFER;
2767
2768 if (!driver_data || !driver_data->verify || !driver_data->init ||
2769 !(driver_data->setpolicy || driver_data->target_index ||
2770 driver_data->target) ||
2771 (driver_data->setpolicy && (driver_data->target_index ||
2772 driver_data->target)) ||
2773 (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2774 (!driver_data->online != !driver_data->offline))
2775 return -EINVAL;
2776
2777 pr_debug("trying to register driver %s\n", driver_data->name);
2778
2779 /* Protect against concurrent CPU online/offline. */
2780 cpus_read_lock();
2781
2782 write_lock_irqsave(&cpufreq_driver_lock, flags);
2783 if (cpufreq_driver) {
2784 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2785 ret = -EEXIST;
2786 goto out;
2787 }
2788 cpufreq_driver = driver_data;
2789 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2790
2791 /*
2792 * Mark support for the scheduler's frequency invariance engine for
2793 * drivers that implement target(), target_index() or fast_switch().
2794 */
2795 if (!cpufreq_driver->setpolicy) {
2796 static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2797 pr_debug("supports frequency invariance");
2798 }
2799
2800 if (driver_data->setpolicy)
2801 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2802
2803 if (cpufreq_boost_supported()) {
2804 ret = create_boost_sysfs_file();
2805 if (ret)
2806 goto err_null_driver;
2807 }
2808
2809 ret = subsys_interface_register(&cpufreq_interface);
2810 if (ret)
2811 goto err_boost_unreg;
2812
2813 if (unlikely(list_empty(&cpufreq_policy_list))) {
2814 /* if all ->init() calls failed, unregister */
2815 ret = -ENODEV;
2816 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2817 driver_data->name);
2818 goto err_if_unreg;
2819 }
2820
2821 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2822 "cpufreq:online",
2823 cpuhp_cpufreq_online,
2824 cpuhp_cpufreq_offline);
2825 if (ret < 0)
2826 goto err_if_unreg;
2827 hp_online = ret;
2828 ret = 0;
2829
2830 pr_debug("driver %s up and running\n", driver_data->name);
2831 goto out;
2832
2833err_if_unreg:
2834 subsys_interface_unregister(&cpufreq_interface);
2835err_boost_unreg:
2836 remove_boost_sysfs_file();
2837err_null_driver:
2838 write_lock_irqsave(&cpufreq_driver_lock, flags);
2839 cpufreq_driver = NULL;
2840 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2841out:
2842 cpus_read_unlock();
2843 return ret;
2844}
2845EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2846
2847/*
2848 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2849 *
2850 * Unregister the current CPUFreq driver. Only call this if you have
2851 * the right to do so, i.e. if you have succeeded in initialising before!
2852 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2853 * currently not initialised.
2854 */
2855int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2856{
2857 unsigned long flags;
2858
2859 if (!cpufreq_driver || (driver != cpufreq_driver))
2860 return -EINVAL;
2861
2862 pr_debug("unregistering driver %s\n", driver->name);
2863
2864 /* Protect against concurrent cpu hotplug */
2865 cpus_read_lock();
2866 subsys_interface_unregister(&cpufreq_interface);
2867 remove_boost_sysfs_file();
2868 static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
2869 cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2870
2871 write_lock_irqsave(&cpufreq_driver_lock, flags);
2872
2873 cpufreq_driver = NULL;
2874
2875 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2876 cpus_read_unlock();
2877
2878 return 0;
2879}
2880EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2881
2882static int __init cpufreq_core_init(void)
2883{
2884 struct cpufreq_governor *gov = cpufreq_default_governor();
2885
2886 if (cpufreq_disabled())
2887 return -ENODEV;
2888
2889 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2890 BUG_ON(!cpufreq_global_kobject);
2891
2892 if (!strlen(default_governor))
2893 strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
2894
2895 return 0;
2896}
2897module_param(off, int, 0444);
2898module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
2899core_initcall(cpufreq_core_init);