Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2019, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17#include <linux/cpu.h>
18#include <linux/module.h>
19#include <linux/slab.h>
20#include <linux/suspend.h>
21#include <asm/cpu_device_id.h>
22#include <asm/intel-family.h>
23
24#define MSR_UNCORE_RATIO_LIMIT 0x620
25#define UNCORE_FREQ_KHZ_MULTIPLIER 100000
26
27/**
28 * struct uncore_data - Encapsulate all uncore data
29 * @stored_uncore_data: Last user changed MSR 620 value, which will be restored
30 * on system resume.
31 * @initial_min_freq_khz: Sampled minimum uncore frequency at driver init
32 * @initial_max_freq_khz: Sampled maximum uncore frequency at driver init
33 * @control_cpu: Designated CPU for a die to read/write
34 * @valid: Mark the data valid/invalid
35 *
36 * This structure is used to encapsulate all data related to uncore sysfs
37 * settings for a die/package.
38 */
39struct uncore_data {
40 struct kobject kobj;
41 struct completion kobj_unregister;
42 u64 stored_uncore_data;
43 u32 initial_min_freq_khz;
44 u32 initial_max_freq_khz;
45 int control_cpu;
46 bool valid;
47};
48
49#define to_uncore_data(a) container_of(a, struct uncore_data, kobj)
50
51/* Max instances for uncore data, one for each die */
52static int uncore_max_entries __read_mostly;
53/* Storage for uncore data for all instances */
54static struct uncore_data *uncore_instances;
55/* Root of the all uncore sysfs kobjs */
56static struct kobject *uncore_root_kobj;
57/* Stores the CPU mask of the target CPUs to use during uncore read/write */
58static cpumask_t uncore_cpu_mask;
59/* CPU online callback register instance */
60static enum cpuhp_state uncore_hp_state __read_mostly;
61/* Mutex to control all mutual exclusions */
62static DEFINE_MUTEX(uncore_lock);
63
64struct uncore_attr {
65 struct attribute attr;
66 ssize_t (*show)(struct kobject *kobj,
67 struct attribute *attr, char *buf);
68 ssize_t (*store)(struct kobject *kobj,
69 struct attribute *attr, const char *c, ssize_t count);
70};
71
72#define define_one_uncore_ro(_name) \
73static struct uncore_attr _name = \
74__ATTR(_name, 0444, show_##_name, NULL)
75
76#define define_one_uncore_rw(_name) \
77static struct uncore_attr _name = \
78__ATTR(_name, 0644, show_##_name, store_##_name)
79
80#define show_uncore_data(member_name) \
81 static ssize_t show_##member_name(struct kobject *kobj, \
82 struct attribute *attr, \
83 char *buf) \
84 { \
85 struct uncore_data *data = to_uncore_data(kobj); \
86 return scnprintf(buf, PAGE_SIZE, "%u\n", \
87 data->member_name); \
88 } \
89 define_one_uncore_ro(member_name)
90
91show_uncore_data(initial_min_freq_khz);
92show_uncore_data(initial_max_freq_khz);
93
94/* Common function to read MSR 0x620 and read min/max */
95static int uncore_read_ratio(struct uncore_data *data, unsigned int *min,
96 unsigned int *max)
97{
98 u64 cap;
99 int ret;
100
101 if (data->control_cpu < 0)
102 return -ENXIO;
103
104 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
105 if (ret)
106 return ret;
107
108 *max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
109 *min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;
110
111 return 0;
112}
113
114/* Common function to set min/max ratios to be used by sysfs callbacks */
115static int uncore_write_ratio(struct uncore_data *data, unsigned int input,
116 int set_max)
117{
118 int ret;
119 u64 cap;
120
121 mutex_lock(&uncore_lock);
122
123 if (data->control_cpu < 0) {
124 ret = -ENXIO;
125 goto finish_write;
126 }
127
128 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
129 if (!input || input > 0x7F) {
130 ret = -EINVAL;
131 goto finish_write;
132 }
133
134 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
135 if (ret)
136 goto finish_write;
137
138 if (set_max) {
139 cap &= ~0x7F;
140 cap |= input;
141 } else {
142 cap &= ~GENMASK(14, 8);
143 cap |= (input << 8);
144 }
145
146 ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
147 if (ret)
148 goto finish_write;
149
150 data->stored_uncore_data = cap;
151
152finish_write:
153 mutex_unlock(&uncore_lock);
154
155 return ret;
156}
157
158static ssize_t store_min_max_freq_khz(struct kobject *kobj,
159 struct attribute *attr,
160 const char *buf, ssize_t count,
161 int min_max)
162{
163 struct uncore_data *data = to_uncore_data(kobj);
164 unsigned int input;
165
166 if (kstrtouint(buf, 10, &input))
167 return -EINVAL;
168
169 uncore_write_ratio(data, input, min_max);
170
171 return count;
172}
173
174static ssize_t show_min_max_freq_khz(struct kobject *kobj,
175 struct attribute *attr,
176 char *buf, int min_max)
177{
178 struct uncore_data *data = to_uncore_data(kobj);
179 unsigned int min, max;
180 int ret;
181
182 mutex_lock(&uncore_lock);
183 ret = uncore_read_ratio(data, &min, &max);
184 mutex_unlock(&uncore_lock);
185 if (ret)
186 return ret;
187
188 if (min_max)
189 return sprintf(buf, "%u\n", max);
190
191 return sprintf(buf, "%u\n", min);
192}
193
194#define store_uncore_min_max(name, min_max) \
195 static ssize_t store_##name(struct kobject *kobj, \
196 struct attribute *attr, \
197 const char *buf, ssize_t count) \
198 { \
199 \
200 return store_min_max_freq_khz(kobj, attr, buf, count, \
201 min_max); \
202 }
203
204#define show_uncore_min_max(name, min_max) \
205 static ssize_t show_##name(struct kobject *kobj, \
206 struct attribute *attr, char *buf) \
207 { \
208 \
209 return show_min_max_freq_khz(kobj, attr, buf, min_max); \
210 }
211
212store_uncore_min_max(min_freq_khz, 0);
213store_uncore_min_max(max_freq_khz, 1);
214
215show_uncore_min_max(min_freq_khz, 0);
216show_uncore_min_max(max_freq_khz, 1);
217
218define_one_uncore_rw(min_freq_khz);
219define_one_uncore_rw(max_freq_khz);
220
221static struct attribute *uncore_attrs[] = {
222 &initial_min_freq_khz.attr,
223 &initial_max_freq_khz.attr,
224 &max_freq_khz.attr,
225 &min_freq_khz.attr,
226 NULL
227};
228
229static void uncore_sysfs_entry_release(struct kobject *kobj)
230{
231 struct uncore_data *data = to_uncore_data(kobj);
232
233 complete(&data->kobj_unregister);
234}
235
236static struct kobj_type uncore_ktype = {
237 .release = uncore_sysfs_entry_release,
238 .sysfs_ops = &kobj_sysfs_ops,
239 .default_attrs = uncore_attrs,
240};
241
242/* Caller provides protection */
243static struct uncore_data *uncore_get_instance(unsigned int cpu)
244{
245 int id = topology_logical_die_id(cpu);
246
247 if (id >= 0 && id < uncore_max_entries)
248 return &uncore_instances[id];
249
250 return NULL;
251}
252
253static void uncore_add_die_entry(int cpu)
254{
255 struct uncore_data *data;
256
257 mutex_lock(&uncore_lock);
258 data = uncore_get_instance(cpu);
259 if (!data) {
260 mutex_unlock(&uncore_lock);
261 return;
262 }
263
264 if (data->valid) {
265 /* control cpu changed */
266 data->control_cpu = cpu;
267 } else {
268 char str[64];
269 int ret;
270
271 memset(data, 0, sizeof(*data));
272 sprintf(str, "package_%02d_die_%02d",
273 topology_physical_package_id(cpu),
274 topology_die_id(cpu));
275
276 uncore_read_ratio(data, &data->initial_min_freq_khz,
277 &data->initial_max_freq_khz);
278
279 init_completion(&data->kobj_unregister);
280
281 ret = kobject_init_and_add(&data->kobj, &uncore_ktype,
282 uncore_root_kobj, str);
283 if (!ret) {
284 data->control_cpu = cpu;
285 data->valid = true;
286 }
287 }
288 mutex_unlock(&uncore_lock);
289}
290
291/* Last CPU in this die is offline, make control cpu invalid */
292static void uncore_remove_die_entry(int cpu)
293{
294 struct uncore_data *data;
295
296 mutex_lock(&uncore_lock);
297 data = uncore_get_instance(cpu);
298 if (data)
299 data->control_cpu = -1;
300 mutex_unlock(&uncore_lock);
301}
302
303static int uncore_event_cpu_online(unsigned int cpu)
304{
305 int target;
306
307 /* Check if there is an online cpu in the package for uncore MSR */
308 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
309 if (target < nr_cpu_ids)
310 return 0;
311
312 /* Use this CPU on this die as a control CPU */
313 cpumask_set_cpu(cpu, &uncore_cpu_mask);
314 uncore_add_die_entry(cpu);
315
316 return 0;
317}
318
319static int uncore_event_cpu_offline(unsigned int cpu)
320{
321 int target;
322
323 /* Check if existing cpu is used for uncore MSRs */
324 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
325 return 0;
326
327 /* Find a new cpu to set uncore MSR */
328 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
329
330 if (target < nr_cpu_ids) {
331 cpumask_set_cpu(target, &uncore_cpu_mask);
332 uncore_add_die_entry(target);
333 } else {
334 uncore_remove_die_entry(cpu);
335 }
336
337 return 0;
338}
339
340static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
341 void *_unused)
342{
343 int cpu;
344
345 switch (mode) {
346 case PM_POST_HIBERNATION:
347 case PM_POST_RESTORE:
348 case PM_POST_SUSPEND:
349 for_each_cpu(cpu, &uncore_cpu_mask) {
350 struct uncore_data *data;
351 int ret;
352
353 data = uncore_get_instance(cpu);
354 if (!data || !data->valid || !data->stored_uncore_data)
355 continue;
356
357 ret = wrmsrl_on_cpu(cpu, MSR_UNCORE_RATIO_LIMIT,
358 data->stored_uncore_data);
359 if (ret)
360 return ret;
361 }
362 break;
363 default:
364 break;
365 }
366 return 0;
367}
368
369static struct notifier_block uncore_pm_nb = {
370 .notifier_call = uncore_pm_notify,
371};
372
373static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
374 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
375 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
376 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
377 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
378 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
379 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
380 {}
381};
382
383static int __init intel_uncore_init(void)
384{
385 const struct x86_cpu_id *id;
386 int ret;
387
388 id = x86_match_cpu(intel_uncore_cpu_ids);
389 if (!id)
390 return -ENODEV;
391
392 uncore_max_entries = topology_max_packages() *
393 topology_max_die_per_package();
394 uncore_instances = kcalloc(uncore_max_entries,
395 sizeof(*uncore_instances), GFP_KERNEL);
396 if (!uncore_instances)
397 return -ENOMEM;
398
399 uncore_root_kobj = kobject_create_and_add("intel_uncore_frequency",
400 &cpu_subsys.dev_root->kobj);
401 if (!uncore_root_kobj) {
402 ret = -ENOMEM;
403 goto err_free;
404 }
405
406 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
407 "platform/x86/uncore-freq:online",
408 uncore_event_cpu_online,
409 uncore_event_cpu_offline);
410 if (ret < 0)
411 goto err_rem_kobj;
412
413 uncore_hp_state = ret;
414
415 ret = register_pm_notifier(&uncore_pm_nb);
416 if (ret)
417 goto err_rem_state;
418
419 return 0;
420
421err_rem_state:
422 cpuhp_remove_state(uncore_hp_state);
423err_rem_kobj:
424 kobject_put(uncore_root_kobj);
425err_free:
426 kfree(uncore_instances);
427
428 return ret;
429}
430module_init(intel_uncore_init)
431
432static void __exit intel_uncore_exit(void)
433{
434 int i;
435
436 unregister_pm_notifier(&uncore_pm_nb);
437 cpuhp_remove_state(uncore_hp_state);
438 for (i = 0; i < uncore_max_entries; ++i) {
439 if (uncore_instances[i].valid) {
440 kobject_put(&uncore_instances[i].kobj);
441 wait_for_completion(&uncore_instances[i].kobj_unregister);
442 }
443 }
444 kobject_put(uncore_root_kobj);
445 kfree(uncore_instances);
446}
447module_exit(intel_uncore_exit)
448
449MODULE_LICENSE("GPL v2");
450MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");