Loading...
Note: File does not exist in v6.9.4.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2024 Google LLC
4 */
5
6#include <linux/arch_topology.h>
7#include <linux/cpufreq.h>
8#include <linux/init.h>
9#include <linux/sched.h>
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/of_address.h>
13#include <linux/of_platform.h>
14#include <linux/platform_device.h>
15#include <linux/slab.h>
16
17/*
18 * CPU0..CPUn
19 * +-------------+-------------------------------+--------+-------+
20 * | Register | Description | Offset | Len |
21 * +-------------+-------------------------------+--------+-------+
22 * | cur_perf | read this register to get | 0x0 | 0x4 |
23 * | | the current perf (integer val | | |
24 * | | representing perf relative to | | |
25 * | | max performance) | | |
26 * | | that vCPU is running at | | |
27 * +-------------+-------------------------------+--------+-------+
28 * | set_perf | write to this register to set | 0x4 | 0x4 |
29 * | | perf value of the vCPU | | |
30 * +-------------+-------------------------------+--------+-------+
31 * | perftbl_len | number of entries in perf | 0x8 | 0x4 |
32 * | | table. A single entry in the | | |
33 * | | perf table denotes no table | | |
34 * | | and the entry contains | | |
35 * | | the maximum perf value | | |
36 * | | that this vCPU supports. | | |
37 * | | The guest can request any | | |
38 * | | value between 1 and max perf | | |
39 * | | when perftbls are not used. | | |
40 * +---------------------------------------------+--------+-------+
41 * | perftbl_sel | write to this register to | 0xc | 0x4 |
42 * | | select perf table entry to | | |
43 * | | read from | | |
44 * +---------------------------------------------+--------+-------+
45 * | perftbl_rd | read this register to get | 0x10 | 0x4 |
46 * | | perf value of the selected | | |
47 * | | entry based on perftbl_sel | | |
48 * +---------------------------------------------+--------+-------+
49 * | perf_domain | performance domain number | 0x14 | 0x4 |
50 * | | that this vCPU belongs to. | | |
51 * | | vCPUs sharing the same perf | | |
52 * | | domain number are part of the | | |
53 * | | same performance domain. | | |
54 * +-------------+-------------------------------+--------+-------+
55 */
56
57#define REG_CUR_PERF_STATE_OFFSET 0x0
58#define REG_SET_PERF_STATE_OFFSET 0x4
59#define REG_PERFTBL_LEN_OFFSET 0x8
60#define REG_PERFTBL_SEL_OFFSET 0xc
61#define REG_PERFTBL_RD_OFFSET 0x10
62#define REG_PERF_DOMAIN_OFFSET 0x14
63#define PER_CPU_OFFSET 0x1000
64
65#define PERFTBL_MAX_ENTRIES 64U
66
67static void __iomem *base;
68static DEFINE_PER_CPU(u32, perftbl_num_entries);
69
70static void virt_scale_freq_tick(void)
71{
72 int cpu = smp_processor_id();
73 u32 max_freq = (u32)cpufreq_get_hw_max_freq(cpu);
74 u64 cur_freq;
75 unsigned long scale;
76
77 cur_freq = (u64)readl_relaxed(base + cpu * PER_CPU_OFFSET
78 + REG_CUR_PERF_STATE_OFFSET);
79
80 cur_freq <<= SCHED_CAPACITY_SHIFT;
81 scale = (unsigned long)div_u64(cur_freq, max_freq);
82 scale = min(scale, SCHED_CAPACITY_SCALE);
83
84 this_cpu_write(arch_freq_scale, scale);
85}
86
87static struct scale_freq_data virt_sfd = {
88 .source = SCALE_FREQ_SOURCE_VIRT,
89 .set_freq_scale = virt_scale_freq_tick,
90};
91
92static unsigned int virt_cpufreq_set_perf(struct cpufreq_policy *policy,
93 unsigned int target_freq)
94{
95 writel_relaxed(target_freq,
96 base + policy->cpu * PER_CPU_OFFSET + REG_SET_PERF_STATE_OFFSET);
97 return 0;
98}
99
100static unsigned int virt_cpufreq_fast_switch(struct cpufreq_policy *policy,
101 unsigned int target_freq)
102{
103 virt_cpufreq_set_perf(policy, target_freq);
104 return target_freq;
105}
106
107static u32 virt_cpufreq_get_perftbl_entry(int cpu, u32 idx)
108{
109 writel_relaxed(idx, base + cpu * PER_CPU_OFFSET +
110 REG_PERFTBL_SEL_OFFSET);
111 return readl_relaxed(base + cpu * PER_CPU_OFFSET +
112 REG_PERFTBL_RD_OFFSET);
113}
114
115static int virt_cpufreq_target(struct cpufreq_policy *policy,
116 unsigned int target_freq,
117 unsigned int relation)
118{
119 struct cpufreq_freqs freqs;
120 int ret = 0;
121
122 freqs.old = policy->cur;
123 freqs.new = target_freq;
124
125 cpufreq_freq_transition_begin(policy, &freqs);
126 ret = virt_cpufreq_set_perf(policy, target_freq);
127 cpufreq_freq_transition_end(policy, &freqs, ret != 0);
128
129 return ret;
130}
131
132static int virt_cpufreq_get_sharing_cpus(struct cpufreq_policy *policy)
133{
134 u32 cur_perf_domain, perf_domain;
135 struct device *cpu_dev;
136 int cpu;
137
138 cur_perf_domain = readl_relaxed(base + policy->cpu *
139 PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET);
140
141 for_each_possible_cpu(cpu) {
142 cpu_dev = get_cpu_device(cpu);
143 if (!cpu_dev)
144 continue;
145
146 perf_domain = readl_relaxed(base + cpu *
147 PER_CPU_OFFSET + REG_PERF_DOMAIN_OFFSET);
148
149 if (perf_domain == cur_perf_domain)
150 cpumask_set_cpu(cpu, policy->cpus);
151 }
152
153 return 0;
154}
155
156static int virt_cpufreq_get_freq_info(struct cpufreq_policy *policy)
157{
158 struct cpufreq_frequency_table *table;
159 u32 num_perftbl_entries, idx;
160
161 num_perftbl_entries = per_cpu(perftbl_num_entries, policy->cpu);
162
163 if (num_perftbl_entries == 1) {
164 policy->cpuinfo.min_freq = 1;
165 policy->cpuinfo.max_freq = virt_cpufreq_get_perftbl_entry(policy->cpu, 0);
166
167 policy->min = policy->cpuinfo.min_freq;
168 policy->max = policy->cpuinfo.max_freq;
169
170 policy->cur = policy->max;
171 return 0;
172 }
173
174 table = kcalloc(num_perftbl_entries + 1, sizeof(*table), GFP_KERNEL);
175 if (!table)
176 return -ENOMEM;
177
178 for (idx = 0; idx < num_perftbl_entries; idx++)
179 table[idx].frequency = virt_cpufreq_get_perftbl_entry(policy->cpu, idx);
180
181 table[idx].frequency = CPUFREQ_TABLE_END;
182 policy->freq_table = table;
183
184 return 0;
185}
186
187static int virt_cpufreq_cpu_init(struct cpufreq_policy *policy)
188{
189 struct device *cpu_dev;
190 int ret;
191
192 cpu_dev = get_cpu_device(policy->cpu);
193 if (!cpu_dev)
194 return -ENODEV;
195
196 ret = virt_cpufreq_get_freq_info(policy);
197 if (ret) {
198 dev_warn(cpu_dev, "failed to get cpufreq info\n");
199 return ret;
200 }
201
202 ret = virt_cpufreq_get_sharing_cpus(policy);
203 if (ret) {
204 dev_warn(cpu_dev, "failed to get sharing cpumask\n");
205 return ret;
206 }
207
208 /*
209 * To simplify and improve latency of handling frequency requests on
210 * the host side, this ensures that the vCPU thread triggering the MMIO
211 * abort is the same thread whose performance constraints (Ex. uclamp
212 * settings) need to be updated. This simplifies the VMM (Virtual
213 * Machine Manager) having to find the correct vCPU thread and/or
214 * facing permission issues when configuring other threads.
215 */
216 policy->dvfs_possible_from_any_cpu = false;
217 policy->fast_switch_possible = true;
218
219 /*
220 * Using the default SCALE_FREQ_SOURCE_CPUFREQ is insufficient since
221 * the actual physical CPU frequency may not match requested frequency
222 * from the vCPU thread due to frequency update latencies or other
223 * inputs to the physical CPU frequency selection. This additional FIE
224 * source allows for more accurate freq_scale updates and only takes
225 * effect if another FIE source such as AMUs have not been registered.
226 */
227 topology_set_scale_freq_source(&virt_sfd, policy->cpus);
228
229 return 0;
230}
231
232static void virt_cpufreq_cpu_exit(struct cpufreq_policy *policy)
233{
234 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_VIRT, policy->related_cpus);
235 kfree(policy->freq_table);
236}
237
238static int virt_cpufreq_online(struct cpufreq_policy *policy)
239{
240 /* Nothing to restore. */
241 return 0;
242}
243
244static int virt_cpufreq_offline(struct cpufreq_policy *policy)
245{
246 /* Dummy offline() to avoid exit() being called and freeing resources. */
247 return 0;
248}
249
250static int virt_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
251{
252 if (policy->freq_table)
253 return cpufreq_frequency_table_verify(policy, policy->freq_table);
254
255 cpufreq_verify_within_cpu_limits(policy);
256 return 0;
257}
258
259static struct cpufreq_driver cpufreq_virt_driver = {
260 .name = "virt-cpufreq",
261 .init = virt_cpufreq_cpu_init,
262 .exit = virt_cpufreq_cpu_exit,
263 .online = virt_cpufreq_online,
264 .offline = virt_cpufreq_offline,
265 .verify = virt_cpufreq_verify_policy,
266 .target = virt_cpufreq_target,
267 .fast_switch = virt_cpufreq_fast_switch,
268 .attr = cpufreq_generic_attr,
269};
270
271static int virt_cpufreq_driver_probe(struct platform_device *pdev)
272{
273 u32 num_perftbl_entries;
274 int ret, cpu;
275
276 base = devm_platform_ioremap_resource(pdev, 0);
277 if (IS_ERR(base))
278 return PTR_ERR(base);
279
280 for_each_possible_cpu(cpu) {
281 num_perftbl_entries = readl_relaxed(base + cpu * PER_CPU_OFFSET +
282 REG_PERFTBL_LEN_OFFSET);
283
284 if (!num_perftbl_entries || num_perftbl_entries > PERFTBL_MAX_ENTRIES)
285 return -ENODEV;
286
287 per_cpu(perftbl_num_entries, cpu) = num_perftbl_entries;
288 }
289
290 ret = cpufreq_register_driver(&cpufreq_virt_driver);
291 if (ret) {
292 dev_err(&pdev->dev, "Virtual CPUFreq driver failed to register: %d\n", ret);
293 return ret;
294 }
295
296 dev_dbg(&pdev->dev, "Virtual CPUFreq driver initialized\n");
297 return 0;
298}
299
300static void virt_cpufreq_driver_remove(struct platform_device *pdev)
301{
302 cpufreq_unregister_driver(&cpufreq_virt_driver);
303}
304
305static const struct of_device_id virt_cpufreq_match[] = {
306 { .compatible = "qemu,virtual-cpufreq", .data = NULL},
307 {}
308};
309MODULE_DEVICE_TABLE(of, virt_cpufreq_match);
310
311static struct platform_driver virt_cpufreq_driver = {
312 .probe = virt_cpufreq_driver_probe,
313 .remove = virt_cpufreq_driver_remove,
314 .driver = {
315 .name = "virt-cpufreq",
316 .of_match_table = virt_cpufreq_match,
317 },
318};
319
320static int __init virt_cpufreq_init(void)
321{
322 return platform_driver_register(&virt_cpufreq_driver);
323}
324postcore_initcall(virt_cpufreq_init);
325
326static void __exit virt_cpufreq_exit(void)
327{
328 platform_driver_unregister(&virt_cpufreq_driver);
329}
330module_exit(virt_cpufreq_exit);
331
332MODULE_DESCRIPTION("Virtual cpufreq driver");
333MODULE_LICENSE("GPL");