1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/syscore_ops.h>
  3#include <linux/suspend.h>
  4#include <linux/cpu.h>
  5
  6#include <asm/msr.h>
  7
  8#define UMWAIT_C02_ENABLE	0
  9
 10#define UMWAIT_CTRL_VAL(max_time, c02_disable)				\
 11	(((max_time) & MSR_IA32_UMWAIT_CONTROL_TIME_MASK) |		\
 12	((c02_disable) & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE))
 13
 14/*
 15 * Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
 16 * umwait max time is 100000 in TSC-quanta and C0.2 is enabled
 17 */
 18static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
 19
 20u32 get_umwait_control_msr(void)
 21{
 22	return umwait_control_cached;
 23}
 24EXPORT_SYMBOL_GPL(get_umwait_control_msr);
 25
 26/*
 27 * Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by
 28 * hardware or BIOS before kernel boot.
 29 */
 30static u32 orig_umwait_control_cached __ro_after_init;
 31
 32/*
 33 * Serialize access to umwait_control_cached and IA32_UMWAIT_CONTROL MSR in
 34 * the sysfs write functions.
 35 */
 36static DEFINE_MUTEX(umwait_lock);
 37
 38static void umwait_update_control_msr(void * unused)
 39{
 40	lockdep_assert_irqs_disabled();
 41	wrmsr(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached), 0);
 42}
 43
 44/*
 45 * The CPU hotplug callback sets the control MSR to the global control
 46 * value.
 47 *
 48 * Disable interrupts so the read of umwait_control_cached and the WRMSR
 49 * are protected against a concurrent sysfs write. Otherwise the sysfs
 50 * write could update the cached value after it had been read on this CPU
 51 * and issue the IPI before the old value had been written. The IPI would
 52 * interrupt, write the new value and after return from IPI the previous
 53 * value would be written by this CPU.
 54 *
 55 * With interrupts disabled the upcoming CPU either sees the new control
 56 * value or the IPI is updating this CPU to the new control value after
 57 * interrupts have been reenabled.
 58 */
 59static int umwait_cpu_online(unsigned int cpu)
 60{
 61	local_irq_disable();
 62	umwait_update_control_msr(NULL);
 63	local_irq_enable();
 64	return 0;
 65}
 66
 67/*
 68 * The CPU hotplug callback sets the control MSR to the original control
 69 * value.
 70 */
 71static int umwait_cpu_offline(unsigned int cpu)
 72{
 73	/*
 74	 * This code is protected by the CPU hotplug already and
 75	 * orig_umwait_control_cached is never changed after it caches
 76	 * the original control MSR value in umwait_init(). So there
 77	 * is no race condition here.
 78	 */
 79	wrmsr(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached, 0);
 80
 81	return 0;
 82}
 83
 84/*
 85 * On resume, restore IA32_UMWAIT_CONTROL MSR on the boot processor which
 86 * is the only active CPU at this time. The MSR is set up on the APs via the
 87 * CPU hotplug callback.
 88 *
 89 * This function is invoked on resume from suspend and hibernation. On
 90 * resume from suspend the restore should be not required, but we neither
 91 * trust the firmware nor does it matter if the same value is written
 92 * again.
 93 */
 94static void umwait_syscore_resume(void)
 95{
 96	umwait_update_control_msr(NULL);
 97}
 98
 99static struct syscore_ops umwait_syscore_ops = {
100	.resume	= umwait_syscore_resume,
101};
102
103/* sysfs interface */
104
105/*
106 * When bit 0 in IA32_UMWAIT_CONTROL MSR is 1, C0.2 is disabled.
107 * Otherwise, C0.2 is enabled.
108 */
109static inline bool umwait_ctrl_c02_enabled(u32 ctrl)
110{
111	return !(ctrl & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE);
112}
113
114static inline u32 umwait_ctrl_max_time(u32 ctrl)
115{
116	return ctrl & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
117}
118
119static inline void umwait_update_control(u32 maxtime, bool c02_enable)
120{
121	u32 ctrl = maxtime & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
122
123	if (!c02_enable)
124		ctrl |= MSR_IA32_UMWAIT_CONTROL_C02_DISABLE;
125
126	WRITE_ONCE(umwait_control_cached, ctrl);
127	/* Propagate to all CPUs */
128	on_each_cpu(umwait_update_control_msr, NULL, 1);
129}
130
131static ssize_t
132enable_c02_show(struct device *dev, struct device_attribute *attr, char *buf)
133{
134	u32 ctrl = READ_ONCE(umwait_control_cached);
135
136	return sprintf(buf, "%d\n", umwait_ctrl_c02_enabled(ctrl));
137}
138
139static ssize_t enable_c02_store(struct device *dev,
140				struct device_attribute *attr,
141				const char *buf, size_t count)
142{
143	bool c02_enable;
144	u32 ctrl;
145	int ret;
146
147	ret = kstrtobool(buf, &c02_enable);
148	if (ret)
149		return ret;
150
151	mutex_lock(&umwait_lock);
152
153	ctrl = READ_ONCE(umwait_control_cached);
154	if (c02_enable != umwait_ctrl_c02_enabled(ctrl))
155		umwait_update_control(ctrl, c02_enable);
156
157	mutex_unlock(&umwait_lock);
158
159	return count;
160}
161static DEVICE_ATTR_RW(enable_c02);
162
163static ssize_t
164max_time_show(struct device *kobj, struct device_attribute *attr, char *buf)
165{
166	u32 ctrl = READ_ONCE(umwait_control_cached);
167
168	return sprintf(buf, "%u\n", umwait_ctrl_max_time(ctrl));
169}
170
171static ssize_t max_time_store(struct device *kobj,
172			      struct device_attribute *attr,
173			      const char *buf, size_t count)
174{
175	u32 max_time, ctrl;
176	int ret;
177
178	ret = kstrtou32(buf, 0, &max_time);
179	if (ret)
180		return ret;
181
182	/* bits[1:0] must be zero */
183	if (max_time & ~MSR_IA32_UMWAIT_CONTROL_TIME_MASK)
184		return -EINVAL;
185
186	mutex_lock(&umwait_lock);
187
188	ctrl = READ_ONCE(umwait_control_cached);
189	if (max_time != umwait_ctrl_max_time(ctrl))
190		umwait_update_control(max_time, umwait_ctrl_c02_enabled(ctrl));
191
192	mutex_unlock(&umwait_lock);
193
194	return count;
195}
196static DEVICE_ATTR_RW(max_time);
197
198static struct attribute *umwait_attrs[] = {
199	&dev_attr_enable_c02.attr,
200	&dev_attr_max_time.attr,
201	NULL
202};
203
204static struct attribute_group umwait_attr_group = {
205	.attrs = umwait_attrs,
206	.name = "umwait_control",
207};
208
209static int __init umwait_init(void)
210{
211	struct device *dev;
212	int ret;
213
214	if (!boot_cpu_has(X86_FEATURE_WAITPKG))
215		return -ENODEV;
216
217	/*
218	 * Cache the original control MSR value before the control MSR is
219	 * changed. This is the only place where orig_umwait_control_cached
220	 * is modified.
221	 */
222	rdmsrl(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
223
224	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online",
225				umwait_cpu_online, umwait_cpu_offline);
226	if (ret < 0) {
227		/*
228		 * On failure, the control MSR on all CPUs has the
229		 * original control value.
230		 */
231		return ret;
232	}
233
234	register_syscore_ops(&umwait_syscore_ops);
235
236	/*
237	 * Add umwait control interface. Ignore failure, so at least the
238	 * default values are set up in case the machine manages to boot.
239	 */
240	dev = cpu_subsys.dev_root;
241	return sysfs_create_group(&dev->kobj, &umwait_attr_group);
242}
243device_initcall(umwait_init);