Loading...
Note: File does not exist in v3.1.
1/*
2 * intel_pstate.c: Native P state management for Intel processors
3 *
4 * (C) Copyright 2012 Intel Corporation
5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 */
12
13#include <linux/kernel.h>
14#include <linux/kernel_stat.h>
15#include <linux/module.h>
16#include <linux/ktime.h>
17#include <linux/hrtimer.h>
18#include <linux/tick.h>
19#include <linux/slab.h>
20#include <linux/sched.h>
21#include <linux/list.h>
22#include <linux/cpu.h>
23#include <linux/cpufreq.h>
24#include <linux/sysfs.h>
25#include <linux/types.h>
26#include <linux/fs.h>
27#include <linux/debugfs.h>
28#include <linux/acpi.h>
29#include <trace/events/power.h>
30
31#include <asm/div64.h>
32#include <asm/msr.h>
33#include <asm/cpu_device_id.h>
34
35#define SAMPLE_COUNT 3
36
37#define BYT_RATIOS 0x66a
38#define BYT_VIDS 0x66b
39#define BYT_TURBO_RATIOS 0x66c
40#define BYT_TURBO_VIDS 0x66d
41
42
43#define FRAC_BITS 8
44#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
45#define fp_toint(X) ((X) >> FRAC_BITS)
46
47
48static inline int32_t mul_fp(int32_t x, int32_t y)
49{
50 return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
51}
52
53static inline int32_t div_fp(int32_t x, int32_t y)
54{
55 return div_s64((int64_t)x << FRAC_BITS, (int64_t)y);
56}
57
58struct sample {
59 int32_t core_pct_busy;
60 u64 aperf;
61 u64 mperf;
62 int freq;
63 ktime_t time;
64};
65
66struct pstate_data {
67 int current_pstate;
68 int min_pstate;
69 int max_pstate;
70 int turbo_pstate;
71};
72
73struct vid_data {
74 int min;
75 int max;
76 int turbo;
77 int32_t ratio;
78};
79
80struct _pid {
81 int setpoint;
82 int32_t integral;
83 int32_t p_gain;
84 int32_t i_gain;
85 int32_t d_gain;
86 int deadband;
87 int32_t last_err;
88};
89
90struct cpudata {
91 int cpu;
92
93 char name[64];
94
95 struct timer_list timer;
96
97 struct pstate_data pstate;
98 struct vid_data vid;
99 struct _pid pid;
100
101 ktime_t last_sample_time;
102 u64 prev_aperf;
103 u64 prev_mperf;
104 struct sample sample;
105};
106
107static struct cpudata **all_cpu_data;
108struct pstate_adjust_policy {
109 int sample_rate_ms;
110 int deadband;
111 int setpoint;
112 int p_gain_pct;
113 int d_gain_pct;
114 int i_gain_pct;
115};
116
117struct pstate_funcs {
118 int (*get_max)(void);
119 int (*get_min)(void);
120 int (*get_turbo)(void);
121 void (*set)(struct cpudata*, int pstate);
122 void (*get_vid)(struct cpudata *);
123};
124
125struct cpu_defaults {
126 struct pstate_adjust_policy pid_policy;
127 struct pstate_funcs funcs;
128};
129
130static struct pstate_adjust_policy pid_params;
131static struct pstate_funcs pstate_funcs;
132
133struct perf_limits {
134 int no_turbo;
135 int max_perf_pct;
136 int min_perf_pct;
137 int32_t max_perf;
138 int32_t min_perf;
139 int max_policy_pct;
140 int max_sysfs_pct;
141};
142
143static struct perf_limits limits = {
144 .no_turbo = 0,
145 .max_perf_pct = 100,
146 .max_perf = int_tofp(1),
147 .min_perf_pct = 0,
148 .min_perf = 0,
149 .max_policy_pct = 100,
150 .max_sysfs_pct = 100,
151};
152
153static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
154 int deadband, int integral) {
155 pid->setpoint = setpoint;
156 pid->deadband = deadband;
157 pid->integral = int_tofp(integral);
158 pid->last_err = int_tofp(setpoint) - int_tofp(busy);
159}
160
161static inline void pid_p_gain_set(struct _pid *pid, int percent)
162{
163 pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
164}
165
166static inline void pid_i_gain_set(struct _pid *pid, int percent)
167{
168 pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
169}
170
171static inline void pid_d_gain_set(struct _pid *pid, int percent)
172{
173
174 pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
175}
176
177static signed int pid_calc(struct _pid *pid, int32_t busy)
178{
179 signed int result;
180 int32_t pterm, dterm, fp_error;
181 int32_t integral_limit;
182
183 fp_error = int_tofp(pid->setpoint) - busy;
184
185 if (abs(fp_error) <= int_tofp(pid->deadband))
186 return 0;
187
188 pterm = mul_fp(pid->p_gain, fp_error);
189
190 pid->integral += fp_error;
191
192 /* limit the integral term */
193 integral_limit = int_tofp(30);
194 if (pid->integral > integral_limit)
195 pid->integral = integral_limit;
196 if (pid->integral < -integral_limit)
197 pid->integral = -integral_limit;
198
199 dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
200 pid->last_err = fp_error;
201
202 result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
203 if (result >= 0)
204 result = result + (1 << (FRAC_BITS-1));
205 else
206 result = result - (1 << (FRAC_BITS-1));
207 return (signed int)fp_toint(result);
208}
209
210static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
211{
212 pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
213 pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
214 pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
215
216 pid_reset(&cpu->pid,
217 pid_params.setpoint,
218 100,
219 pid_params.deadband,
220 0);
221}
222
223static inline void intel_pstate_reset_all_pid(void)
224{
225 unsigned int cpu;
226 for_each_online_cpu(cpu) {
227 if (all_cpu_data[cpu])
228 intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
229 }
230}
231
232/************************** debugfs begin ************************/
233static int pid_param_set(void *data, u64 val)
234{
235 *(u32 *)data = val;
236 intel_pstate_reset_all_pid();
237 return 0;
238}
239static int pid_param_get(void *data, u64 *val)
240{
241 *val = *(u32 *)data;
242 return 0;
243}
244DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get,
245 pid_param_set, "%llu\n");
246
247struct pid_param {
248 char *name;
249 void *value;
250};
251
252static struct pid_param pid_files[] = {
253 {"sample_rate_ms", &pid_params.sample_rate_ms},
254 {"d_gain_pct", &pid_params.d_gain_pct},
255 {"i_gain_pct", &pid_params.i_gain_pct},
256 {"deadband", &pid_params.deadband},
257 {"setpoint", &pid_params.setpoint},
258 {"p_gain_pct", &pid_params.p_gain_pct},
259 {NULL, NULL}
260};
261
262static struct dentry *debugfs_parent;
263static void intel_pstate_debug_expose_params(void)
264{
265 int i = 0;
266
267 debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
268 if (IS_ERR_OR_NULL(debugfs_parent))
269 return;
270 while (pid_files[i].name) {
271 debugfs_create_file(pid_files[i].name, 0660,
272 debugfs_parent, pid_files[i].value,
273 &fops_pid_param);
274 i++;
275 }
276}
277
278/************************** debugfs end ************************/
279
280/************************** sysfs begin ************************/
281#define show_one(file_name, object) \
282 static ssize_t show_##file_name \
283 (struct kobject *kobj, struct attribute *attr, char *buf) \
284 { \
285 return sprintf(buf, "%u\n", limits.object); \
286 }
287
288static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
289 const char *buf, size_t count)
290{
291 unsigned int input;
292 int ret;
293 ret = sscanf(buf, "%u", &input);
294 if (ret != 1)
295 return -EINVAL;
296 limits.no_turbo = clamp_t(int, input, 0 , 1);
297
298 return count;
299}
300
301static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
302 const char *buf, size_t count)
303{
304 unsigned int input;
305 int ret;
306 ret = sscanf(buf, "%u", &input);
307 if (ret != 1)
308 return -EINVAL;
309
310 limits.max_sysfs_pct = clamp_t(int, input, 0 , 100);
311 limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
312 limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
313 return count;
314}
315
316static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
317 const char *buf, size_t count)
318{
319 unsigned int input;
320 int ret;
321 ret = sscanf(buf, "%u", &input);
322 if (ret != 1)
323 return -EINVAL;
324 limits.min_perf_pct = clamp_t(int, input, 0 , 100);
325 limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
326
327 return count;
328}
329
330show_one(no_turbo, no_turbo);
331show_one(max_perf_pct, max_perf_pct);
332show_one(min_perf_pct, min_perf_pct);
333
334define_one_global_rw(no_turbo);
335define_one_global_rw(max_perf_pct);
336define_one_global_rw(min_perf_pct);
337
338static struct attribute *intel_pstate_attributes[] = {
339 &no_turbo.attr,
340 &max_perf_pct.attr,
341 &min_perf_pct.attr,
342 NULL
343};
344
345static struct attribute_group intel_pstate_attr_group = {
346 .attrs = intel_pstate_attributes,
347};
348static struct kobject *intel_pstate_kobject;
349
350static void intel_pstate_sysfs_expose_params(void)
351{
352 int rc;
353
354 intel_pstate_kobject = kobject_create_and_add("intel_pstate",
355 &cpu_subsys.dev_root->kobj);
356 BUG_ON(!intel_pstate_kobject);
357 rc = sysfs_create_group(intel_pstate_kobject,
358 &intel_pstate_attr_group);
359 BUG_ON(rc);
360}
361
362/************************** sysfs end ************************/
363static int byt_get_min_pstate(void)
364{
365 u64 value;
366 rdmsrl(BYT_RATIOS, value);
367 return (value >> 8) & 0x3F;
368}
369
370static int byt_get_max_pstate(void)
371{
372 u64 value;
373 rdmsrl(BYT_RATIOS, value);
374 return (value >> 16) & 0x3F;
375}
376
377static int byt_get_turbo_pstate(void)
378{
379 u64 value;
380 rdmsrl(BYT_TURBO_RATIOS, value);
381 return value & 0x3F;
382}
383
384static void byt_set_pstate(struct cpudata *cpudata, int pstate)
385{
386 u64 val;
387 int32_t vid_fp;
388 u32 vid;
389
390 val = pstate << 8;
391 if (limits.no_turbo)
392 val |= (u64)1 << 32;
393
394 vid_fp = cpudata->vid.min + mul_fp(
395 int_tofp(pstate - cpudata->pstate.min_pstate),
396 cpudata->vid.ratio);
397
398 vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
399 vid = fp_toint(vid_fp);
400
401 if (pstate > cpudata->pstate.max_pstate)
402 vid = cpudata->vid.turbo;
403
404 val |= vid;
405
406 wrmsrl(MSR_IA32_PERF_CTL, val);
407}
408
409static void byt_get_vid(struct cpudata *cpudata)
410{
411 u64 value;
412
413
414 rdmsrl(BYT_VIDS, value);
415 cpudata->vid.min = int_tofp((value >> 8) & 0x3f);
416 cpudata->vid.max = int_tofp((value >> 16) & 0x3f);
417 cpudata->vid.ratio = div_fp(
418 cpudata->vid.max - cpudata->vid.min,
419 int_tofp(cpudata->pstate.max_pstate -
420 cpudata->pstate.min_pstate));
421
422 rdmsrl(BYT_TURBO_VIDS, value);
423 cpudata->vid.turbo = value & 0x7f;
424}
425
426
427static int core_get_min_pstate(void)
428{
429 u64 value;
430 rdmsrl(MSR_PLATFORM_INFO, value);
431 return (value >> 40) & 0xFF;
432}
433
434static int core_get_max_pstate(void)
435{
436 u64 value;
437 rdmsrl(MSR_PLATFORM_INFO, value);
438 return (value >> 8) & 0xFF;
439}
440
441static int core_get_turbo_pstate(void)
442{
443 u64 value;
444 int nont, ret;
445 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
446 nont = core_get_max_pstate();
447 ret = ((value) & 255);
448 if (ret <= nont)
449 ret = nont;
450 return ret;
451}
452
453static void core_set_pstate(struct cpudata *cpudata, int pstate)
454{
455 u64 val;
456
457 val = pstate << 8;
458 if (limits.no_turbo)
459 val |= (u64)1 << 32;
460
461 wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
462}
463
464static struct cpu_defaults core_params = {
465 .pid_policy = {
466 .sample_rate_ms = 10,
467 .deadband = 0,
468 .setpoint = 97,
469 .p_gain_pct = 20,
470 .d_gain_pct = 0,
471 .i_gain_pct = 0,
472 },
473 .funcs = {
474 .get_max = core_get_max_pstate,
475 .get_min = core_get_min_pstate,
476 .get_turbo = core_get_turbo_pstate,
477 .set = core_set_pstate,
478 },
479};
480
481static struct cpu_defaults byt_params = {
482 .pid_policy = {
483 .sample_rate_ms = 10,
484 .deadband = 0,
485 .setpoint = 97,
486 .p_gain_pct = 14,
487 .d_gain_pct = 0,
488 .i_gain_pct = 4,
489 },
490 .funcs = {
491 .get_max = byt_get_max_pstate,
492 .get_min = byt_get_min_pstate,
493 .get_turbo = byt_get_turbo_pstate,
494 .set = byt_set_pstate,
495 .get_vid = byt_get_vid,
496 },
497};
498
499
500static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
501{
502 int max_perf = cpu->pstate.turbo_pstate;
503 int max_perf_adj;
504 int min_perf;
505 if (limits.no_turbo)
506 max_perf = cpu->pstate.max_pstate;
507
508 max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits.max_perf));
509 *max = clamp_t(int, max_perf_adj,
510 cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
511
512 min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits.min_perf));
513 *min = clamp_t(int, min_perf,
514 cpu->pstate.min_pstate, max_perf);
515}
516
517static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
518{
519 int max_perf, min_perf;
520
521 intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
522
523 pstate = clamp_t(int, pstate, min_perf, max_perf);
524
525 if (pstate == cpu->pstate.current_pstate)
526 return;
527
528 trace_cpu_frequency(pstate * 100000, cpu->cpu);
529
530 cpu->pstate.current_pstate = pstate;
531
532 pstate_funcs.set(cpu, pstate);
533}
534
535static inline void intel_pstate_pstate_increase(struct cpudata *cpu, int steps)
536{
537 int target;
538 target = cpu->pstate.current_pstate + steps;
539
540 intel_pstate_set_pstate(cpu, target);
541}
542
543static inline void intel_pstate_pstate_decrease(struct cpudata *cpu, int steps)
544{
545 int target;
546 target = cpu->pstate.current_pstate - steps;
547 intel_pstate_set_pstate(cpu, target);
548}
549
550static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
551{
552 sprintf(cpu->name, "Intel 2nd generation core");
553
554 cpu->pstate.min_pstate = pstate_funcs.get_min();
555 cpu->pstate.max_pstate = pstate_funcs.get_max();
556 cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
557
558 if (pstate_funcs.get_vid)
559 pstate_funcs.get_vid(cpu);
560 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
561}
562
563static inline void intel_pstate_calc_busy(struct cpudata *cpu,
564 struct sample *sample)
565{
566 int64_t core_pct;
567 int32_t rem;
568
569 core_pct = int_tofp(sample->aperf) * int_tofp(100);
570 core_pct = div_u64_rem(core_pct, int_tofp(sample->mperf), &rem);
571
572 if ((rem << 1) >= int_tofp(sample->mperf))
573 core_pct += 1;
574
575 sample->freq = fp_toint(
576 mul_fp(int_tofp(cpu->pstate.max_pstate * 1000), core_pct));
577
578 sample->core_pct_busy = (int32_t)core_pct;
579}
580
581static inline void intel_pstate_sample(struct cpudata *cpu)
582{
583 u64 aperf, mperf;
584
585 rdmsrl(MSR_IA32_APERF, aperf);
586 rdmsrl(MSR_IA32_MPERF, mperf);
587
588 aperf = aperf >> FRAC_BITS;
589 mperf = mperf >> FRAC_BITS;
590
591 cpu->last_sample_time = cpu->sample.time;
592 cpu->sample.time = ktime_get();
593 cpu->sample.aperf = aperf;
594 cpu->sample.mperf = mperf;
595 cpu->sample.aperf -= cpu->prev_aperf;
596 cpu->sample.mperf -= cpu->prev_mperf;
597
598 intel_pstate_calc_busy(cpu, &cpu->sample);
599
600 cpu->prev_aperf = aperf;
601 cpu->prev_mperf = mperf;
602}
603
604static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
605{
606 int sample_time, delay;
607
608 sample_time = pid_params.sample_rate_ms;
609 delay = msecs_to_jiffies(sample_time);
610 mod_timer_pinned(&cpu->timer, jiffies + delay);
611}
612
613static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
614{
615 int32_t core_busy, max_pstate, current_pstate, sample_ratio;
616 u32 duration_us;
617 u32 sample_time;
618
619 core_busy = cpu->sample.core_pct_busy;
620 max_pstate = int_tofp(cpu->pstate.max_pstate);
621 current_pstate = int_tofp(cpu->pstate.current_pstate);
622 core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
623
624 sample_time = (pid_params.sample_rate_ms * USEC_PER_MSEC);
625 duration_us = (u32) ktime_us_delta(cpu->sample.time,
626 cpu->last_sample_time);
627 if (duration_us > sample_time * 3) {
628 sample_ratio = div_fp(int_tofp(sample_time),
629 int_tofp(duration_us));
630 core_busy = mul_fp(core_busy, sample_ratio);
631 }
632
633 return core_busy;
634}
635
636static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
637{
638 int32_t busy_scaled;
639 struct _pid *pid;
640 signed int ctl = 0;
641 int steps;
642
643 pid = &cpu->pid;
644 busy_scaled = intel_pstate_get_scaled_busy(cpu);
645
646 ctl = pid_calc(pid, busy_scaled);
647
648 steps = abs(ctl);
649
650 if (ctl < 0)
651 intel_pstate_pstate_increase(cpu, steps);
652 else
653 intel_pstate_pstate_decrease(cpu, steps);
654}
655
656static void intel_pstate_timer_func(unsigned long __data)
657{
658 struct cpudata *cpu = (struct cpudata *) __data;
659 struct sample *sample;
660
661 intel_pstate_sample(cpu);
662
663 sample = &cpu->sample;
664
665 intel_pstate_adjust_busy_pstate(cpu);
666
667 trace_pstate_sample(fp_toint(sample->core_pct_busy),
668 fp_toint(intel_pstate_get_scaled_busy(cpu)),
669 cpu->pstate.current_pstate,
670 sample->mperf,
671 sample->aperf,
672 sample->freq);
673
674 intel_pstate_set_sample_time(cpu);
675}
676
677#define ICPU(model, policy) \
678 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
679 (unsigned long)&policy }
680
681static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
682 ICPU(0x2a, core_params),
683 ICPU(0x2d, core_params),
684 ICPU(0x37, byt_params),
685 ICPU(0x3a, core_params),
686 ICPU(0x3c, core_params),
687 ICPU(0x3e, core_params),
688 ICPU(0x3f, core_params),
689 ICPU(0x45, core_params),
690 ICPU(0x46, core_params),
691 {}
692};
693MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
694
695static int intel_pstate_init_cpu(unsigned int cpunum)
696{
697
698 const struct x86_cpu_id *id;
699 struct cpudata *cpu;
700
701 id = x86_match_cpu(intel_pstate_cpu_ids);
702 if (!id)
703 return -ENODEV;
704
705 all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), GFP_KERNEL);
706 if (!all_cpu_data[cpunum])
707 return -ENOMEM;
708
709 cpu = all_cpu_data[cpunum];
710
711 intel_pstate_get_cpu_pstates(cpu);
712
713 cpu->cpu = cpunum;
714
715 init_timer_deferrable(&cpu->timer);
716 cpu->timer.function = intel_pstate_timer_func;
717 cpu->timer.data =
718 (unsigned long)cpu;
719 cpu->timer.expires = jiffies + HZ/100;
720 intel_pstate_busy_pid_reset(cpu);
721 intel_pstate_sample(cpu);
722
723 add_timer_on(&cpu->timer, cpunum);
724
725 pr_info("Intel pstate controlling: cpu %d\n", cpunum);
726
727 return 0;
728}
729
730static unsigned int intel_pstate_get(unsigned int cpu_num)
731{
732 struct sample *sample;
733 struct cpudata *cpu;
734
735 cpu = all_cpu_data[cpu_num];
736 if (!cpu)
737 return 0;
738 sample = &cpu->sample;
739 return sample->freq;
740}
741
742static int intel_pstate_set_policy(struct cpufreq_policy *policy)
743{
744 struct cpudata *cpu;
745
746 cpu = all_cpu_data[policy->cpu];
747
748 if (!policy->cpuinfo.max_freq)
749 return -ENODEV;
750
751 if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
752 limits.min_perf_pct = 100;
753 limits.min_perf = int_tofp(1);
754 limits.max_perf_pct = 100;
755 limits.max_perf = int_tofp(1);
756 limits.no_turbo = 0;
757 return 0;
758 }
759 limits.min_perf_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
760 limits.min_perf_pct = clamp_t(int, limits.min_perf_pct, 0 , 100);
761 limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100));
762
763 limits.max_policy_pct = policy->max * 100 / policy->cpuinfo.max_freq;
764 limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100);
765 limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct);
766 limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100));
767
768 return 0;
769}
770
771static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
772{
773 cpufreq_verify_within_cpu_limits(policy);
774
775 if ((policy->policy != CPUFREQ_POLICY_POWERSAVE) &&
776 (policy->policy != CPUFREQ_POLICY_PERFORMANCE))
777 return -EINVAL;
778
779 return 0;
780}
781
782static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
783{
784 int cpu_num = policy->cpu;
785 struct cpudata *cpu = all_cpu_data[cpu_num];
786
787 pr_info("intel_pstate CPU %d exiting\n", cpu_num);
788
789 del_timer_sync(&all_cpu_data[cpu_num]->timer);
790 intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
791 kfree(all_cpu_data[cpu_num]);
792 all_cpu_data[cpu_num] = NULL;
793}
794
795static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
796{
797 struct cpudata *cpu;
798 int rc;
799
800 rc = intel_pstate_init_cpu(policy->cpu);
801 if (rc)
802 return rc;
803
804 cpu = all_cpu_data[policy->cpu];
805
806 if (!limits.no_turbo &&
807 limits.min_perf_pct == 100 && limits.max_perf_pct == 100)
808 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
809 else
810 policy->policy = CPUFREQ_POLICY_POWERSAVE;
811
812 policy->min = cpu->pstate.min_pstate * 100000;
813 policy->max = cpu->pstate.turbo_pstate * 100000;
814
815 /* cpuinfo and default policy values */
816 policy->cpuinfo.min_freq = cpu->pstate.min_pstate * 100000;
817 policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate * 100000;
818 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
819 cpumask_set_cpu(policy->cpu, policy->cpus);
820
821 return 0;
822}
823
824static struct cpufreq_driver intel_pstate_driver = {
825 .flags = CPUFREQ_CONST_LOOPS,
826 .verify = intel_pstate_verify_policy,
827 .setpolicy = intel_pstate_set_policy,
828 .get = intel_pstate_get,
829 .init = intel_pstate_cpu_init,
830 .stop_cpu = intel_pstate_stop_cpu,
831 .name = "intel_pstate",
832};
833
834static int __initdata no_load;
835
836static int intel_pstate_msrs_not_valid(void)
837{
838 /* Check that all the msr's we are using are valid. */
839 u64 aperf, mperf, tmp;
840
841 rdmsrl(MSR_IA32_APERF, aperf);
842 rdmsrl(MSR_IA32_MPERF, mperf);
843
844 if (!pstate_funcs.get_max() ||
845 !pstate_funcs.get_min() ||
846 !pstate_funcs.get_turbo())
847 return -ENODEV;
848
849 rdmsrl(MSR_IA32_APERF, tmp);
850 if (!(tmp - aperf))
851 return -ENODEV;
852
853 rdmsrl(MSR_IA32_MPERF, tmp);
854 if (!(tmp - mperf))
855 return -ENODEV;
856
857 return 0;
858}
859
860static void copy_pid_params(struct pstate_adjust_policy *policy)
861{
862 pid_params.sample_rate_ms = policy->sample_rate_ms;
863 pid_params.p_gain_pct = policy->p_gain_pct;
864 pid_params.i_gain_pct = policy->i_gain_pct;
865 pid_params.d_gain_pct = policy->d_gain_pct;
866 pid_params.deadband = policy->deadband;
867 pid_params.setpoint = policy->setpoint;
868}
869
870static void copy_cpu_funcs(struct pstate_funcs *funcs)
871{
872 pstate_funcs.get_max = funcs->get_max;
873 pstate_funcs.get_min = funcs->get_min;
874 pstate_funcs.get_turbo = funcs->get_turbo;
875 pstate_funcs.set = funcs->set;
876 pstate_funcs.get_vid = funcs->get_vid;
877}
878
879#if IS_ENABLED(CONFIG_ACPI)
880#include <acpi/processor.h>
881
882static bool intel_pstate_no_acpi_pss(void)
883{
884 int i;
885
886 for_each_possible_cpu(i) {
887 acpi_status status;
888 union acpi_object *pss;
889 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
890 struct acpi_processor *pr = per_cpu(processors, i);
891
892 if (!pr)
893 continue;
894
895 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
896 if (ACPI_FAILURE(status))
897 continue;
898
899 pss = buffer.pointer;
900 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
901 kfree(pss);
902 return false;
903 }
904
905 kfree(pss);
906 }
907
908 return true;
909}
910
911struct hw_vendor_info {
912 u16 valid;
913 char oem_id[ACPI_OEM_ID_SIZE];
914 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
915};
916
917/* Hardware vendor-specific info that has its own power management modes */
918static struct hw_vendor_info vendor_info[] = {
919 {1, "HP ", "ProLiant"},
920 {0, "", ""},
921};
922
923static bool intel_pstate_platform_pwr_mgmt_exists(void)
924{
925 struct acpi_table_header hdr;
926 struct hw_vendor_info *v_info;
927
928 if (acpi_disabled
929 || ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
930 return false;
931
932 for (v_info = vendor_info; v_info->valid; v_info++) {
933 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE)
934 && !strncmp(hdr.oem_table_id, v_info->oem_table_id, ACPI_OEM_TABLE_ID_SIZE)
935 && intel_pstate_no_acpi_pss())
936 return true;
937 }
938
939 return false;
940}
941#else /* CONFIG_ACPI not enabled */
942static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
943#endif /* CONFIG_ACPI */
944
945static int __init intel_pstate_init(void)
946{
947 int cpu, rc = 0;
948 const struct x86_cpu_id *id;
949 struct cpu_defaults *cpu_info;
950
951 if (no_load)
952 return -ENODEV;
953
954 id = x86_match_cpu(intel_pstate_cpu_ids);
955 if (!id)
956 return -ENODEV;
957
958 /*
959 * The Intel pstate driver will be ignored if the platform
960 * firmware has its own power management modes.
961 */
962 if (intel_pstate_platform_pwr_mgmt_exists())
963 return -ENODEV;
964
965 cpu_info = (struct cpu_defaults *)id->driver_data;
966
967 copy_pid_params(&cpu_info->pid_policy);
968 copy_cpu_funcs(&cpu_info->funcs);
969
970 if (intel_pstate_msrs_not_valid())
971 return -ENODEV;
972
973 pr_info("Intel P-state driver initializing.\n");
974
975 all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
976 if (!all_cpu_data)
977 return -ENOMEM;
978
979 rc = cpufreq_register_driver(&intel_pstate_driver);
980 if (rc)
981 goto out;
982
983 intel_pstate_debug_expose_params();
984 intel_pstate_sysfs_expose_params();
985
986 return rc;
987out:
988 get_online_cpus();
989 for_each_online_cpu(cpu) {
990 if (all_cpu_data[cpu]) {
991 del_timer_sync(&all_cpu_data[cpu]->timer);
992 kfree(all_cpu_data[cpu]);
993 }
994 }
995
996 put_online_cpus();
997 vfree(all_cpu_data);
998 return -ENODEV;
999}
1000device_initcall(intel_pstate_init);
1001
1002static int __init intel_pstate_setup(char *str)
1003{
1004 if (!str)
1005 return -EINVAL;
1006
1007 if (!strcmp(str, "disable"))
1008 no_load = 1;
1009 return 0;
1010}
1011early_param("intel_pstate", intel_pstate_setup);
1012
1013MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1014MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1015MODULE_LICENSE("GPL");