1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * acpi_pad.c ACPI Processor Aggregator Driver
  4 *
  5 * Copyright (c) 2009, Intel Corporation.
 
 
 
 
 
 
 
 
 
 
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/cpumask.h>
 10#include <linux/module.h>
 11#include <linux/init.h>
 12#include <linux/types.h>
 13#include <linux/kthread.h>
 14#include <uapi/linux/sched/types.h>
 15#include <linux/freezer.h>
 16#include <linux/cpu.h>
 17#include <linux/tick.h>
 18#include <linux/slab.h>
 19#include <linux/acpi.h>
 20#include <linux/perf_event.h>
 21#include <linux/platform_device.h>
 22#include <asm/mwait.h>
 23#include <xen/xen.h>
 24
 25#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
 26#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 27#define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 28
 29#define ACPI_PROCESSOR_AGGREGATOR_STATUS_SUCCESS	0
 30#define ACPI_PROCESSOR_AGGREGATOR_STATUS_NO_ACTION	1
 31
 32static DEFINE_MUTEX(isolated_cpus_lock);
 33static DEFINE_MUTEX(round_robin_lock);
 34
 35static unsigned long power_saving_mwait_eax;
 36
 37static unsigned char tsc_detected_unstable;
 38static unsigned char tsc_marked_unstable;
 39
 40static void power_saving_mwait_init(void)
 41{
 42	unsigned int eax, ebx, ecx, edx;
 43	unsigned int highest_cstate = 0;
 44	unsigned int highest_subcstate = 0;
 45	int i;
 46
 47	if (!boot_cpu_has(X86_FEATURE_MWAIT))
 48		return;
 49	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
 50		return;
 51
 52	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
 53
 54	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
 55	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
 56		return;
 57
 58	edx >>= MWAIT_SUBSTATE_SIZE;
 59	for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
 60		if (edx & MWAIT_SUBSTATE_MASK) {
 61			highest_cstate = i;
 62			highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
 63		}
 64	}
 65	power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
 66		(highest_subcstate - 1);
 67
 68#if defined(CONFIG_X86)
 69	switch (boot_cpu_data.x86_vendor) {
 70	case X86_VENDOR_HYGON:
 71	case X86_VENDOR_AMD:
 72	case X86_VENDOR_INTEL:
 73	case X86_VENDOR_ZHAOXIN:
 74	case X86_VENDOR_CENTAUR:
 75		/*
 76		 * AMD Fam10h TSC will tick in all
 77		 * C/P/S0/S1 states when this bit is set.
 78		 */
 79		if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 80			tsc_detected_unstable = 1;
 81		break;
 82	default:
 83		/* TSC could halt in idle */
 84		tsc_detected_unstable = 1;
 85	}
 86#endif
 87}
 88
 89static unsigned long cpu_weight[NR_CPUS];
 90static int tsk_in_cpu[NR_CPUS] = {[0 ... NR_CPUS-1] = -1};
 91static DECLARE_BITMAP(pad_busy_cpus_bits, NR_CPUS);
 92static void round_robin_cpu(unsigned int tsk_index)
 93{
 94	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
 95	cpumask_var_t tmp;
 96	int cpu;
 97	unsigned long min_weight = -1;
 98	unsigned long preferred_cpu;
 99
100	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
101		return;
102
103	mutex_lock(&round_robin_lock);
104	cpumask_clear(tmp);
105	for_each_cpu(cpu, pad_busy_cpus)
106		cpumask_or(tmp, tmp, topology_sibling_cpumask(cpu));
107	cpumask_andnot(tmp, cpu_online_mask, tmp);
108	/* avoid HT siblings if possible */
109	if (cpumask_empty(tmp))
110		cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
111	if (cpumask_empty(tmp)) {
112		mutex_unlock(&round_robin_lock);
113		free_cpumask_var(tmp);
114		return;
115	}
116	for_each_cpu(cpu, tmp) {
117		if (cpu_weight[cpu] < min_weight) {
118			min_weight = cpu_weight[cpu];
119			preferred_cpu = cpu;
120		}
121	}
122
123	if (tsk_in_cpu[tsk_index] != -1)
124		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
125	tsk_in_cpu[tsk_index] = preferred_cpu;
126	cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
127	cpu_weight[preferred_cpu]++;
128	mutex_unlock(&round_robin_lock);
129
130	set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
131
132	free_cpumask_var(tmp);
133}
134
135static void exit_round_robin(unsigned int tsk_index)
136{
137	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
138
139	if (tsk_in_cpu[tsk_index] != -1) {
140		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
141		tsk_in_cpu[tsk_index] = -1;
142	}
143}
144
145static unsigned int idle_pct = 5; /* percentage */
146static unsigned int round_robin_time = 1; /* second */
147static int power_saving_thread(void *data)
148{
 
149	int do_sleep;
150	unsigned int tsk_index = (unsigned long)data;
151	u64 last_jiffies = 0;
152
153	sched_set_fifo_low(current);
154
155	while (!kthread_should_stop()) {
156		unsigned long expire_time;
157
158		/* round robin to cpus */
159		expire_time = last_jiffies + round_robin_time * HZ;
160		if (time_before(expire_time, jiffies)) {
161			last_jiffies = jiffies;
162			round_robin_cpu(tsk_index);
163		}
164
165		do_sleep = 0;
166
167		expire_time = jiffies + HZ * (100 - idle_pct) / 100;
168
169		while (!need_resched()) {
170			if (tsc_detected_unstable && !tsc_marked_unstable) {
171				/* TSC could halt in idle, so notify users */
172				mark_tsc_unstable("TSC halts in idle");
173				tsc_marked_unstable = 1;
174			}
175			local_irq_disable();
176
177			perf_lopwr_cb(true);
178
179			tick_broadcast_enable();
180			tick_broadcast_enter();
181			stop_critical_timings();
182
183			mwait_idle_with_hints(power_saving_mwait_eax, 1);
184
185			start_critical_timings();
186			tick_broadcast_exit();
187
188			perf_lopwr_cb(false);
189
190			local_irq_enable();
191
192			if (time_before(expire_time, jiffies)) {
193				do_sleep = 1;
194				break;
195			}
196		}
197
198		/*
199		 * current sched_rt has threshold for rt task running time.
200		 * When a rt task uses 95% CPU time, the rt thread will be
201		 * scheduled out for 5% CPU time to not starve other tasks. But
202		 * the mechanism only works when all CPUs have RT task running,
203		 * as if one CPU hasn't RT task, RT task from other CPUs will
204		 * borrow CPU time from this CPU and cause RT task use > 95%
205		 * CPU time. To make 'avoid starvation' work, takes a nap here.
206		 */
207		if (unlikely(do_sleep))
208			schedule_timeout_killable(HZ * idle_pct / 100);
209
210		/* If an external event has set the need_resched flag, then
211		 * we need to deal with it, or this loop will continue to
212		 * spin without calling __mwait().
213		 */
214		if (unlikely(need_resched()))
215			schedule();
216	}
217
218	exit_round_robin(tsk_index);
219	return 0;
220}
221
222static struct task_struct *ps_tsks[NR_CPUS];
223static unsigned int ps_tsk_num;
224static int create_power_saving_task(void)
225{
226	int rc;
227
228	ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
229		(void *)(unsigned long)ps_tsk_num,
230		"acpi_pad/%d", ps_tsk_num);
231
232	if (IS_ERR(ps_tsks[ps_tsk_num])) {
233		rc = PTR_ERR(ps_tsks[ps_tsk_num]);
234		ps_tsks[ps_tsk_num] = NULL;
235	} else {
236		rc = 0;
237		ps_tsk_num++;
238	}
239
240	return rc;
241}
242
243static void destroy_power_saving_task(void)
244{
245	if (ps_tsk_num > 0) {
246		ps_tsk_num--;
247		kthread_stop(ps_tsks[ps_tsk_num]);
248		ps_tsks[ps_tsk_num] = NULL;
249	}
250}
251
252static void set_power_saving_task_num(unsigned int num)
253{
254	if (num > ps_tsk_num) {
255		while (ps_tsk_num < num) {
256			if (create_power_saving_task())
257				return;
258		}
259	} else if (num < ps_tsk_num) {
260		while (ps_tsk_num > num)
261			destroy_power_saving_task();
262	}
263}
264
265static void acpi_pad_idle_cpus(unsigned int num_cpus)
266{
267	cpus_read_lock();
268
269	num_cpus = min_t(unsigned int, num_cpus, num_online_cpus());
270	set_power_saving_task_num(num_cpus);
271
272	cpus_read_unlock();
273}
274
275static uint32_t acpi_pad_idle_cpus_num(void)
276{
277	return ps_tsk_num;
278}
279
280static ssize_t rrtime_store(struct device *dev,
281	struct device_attribute *attr, const char *buf, size_t count)
282{
283	unsigned long num;
284
285	if (kstrtoul(buf, 0, &num))
286		return -EINVAL;
287	if (num < 1 || num >= 100)
288		return -EINVAL;
289	mutex_lock(&isolated_cpus_lock);
290	round_robin_time = num;
291	mutex_unlock(&isolated_cpus_lock);
292	return count;
293}
294
295static ssize_t rrtime_show(struct device *dev,
296	struct device_attribute *attr, char *buf)
297{
298	return sysfs_emit(buf, "%d\n", round_robin_time);
299}
300static DEVICE_ATTR_RW(rrtime);
 
 
301
302static ssize_t idlepct_store(struct device *dev,
303	struct device_attribute *attr, const char *buf, size_t count)
304{
305	unsigned long num;
306
307	if (kstrtoul(buf, 0, &num))
308		return -EINVAL;
309	if (num < 1 || num >= 100)
310		return -EINVAL;
311	mutex_lock(&isolated_cpus_lock);
312	idle_pct = num;
313	mutex_unlock(&isolated_cpus_lock);
314	return count;
315}
316
317static ssize_t idlepct_show(struct device *dev,
318	struct device_attribute *attr, char *buf)
319{
320	return sysfs_emit(buf, "%d\n", idle_pct);
321}
322static DEVICE_ATTR_RW(idlepct);
 
 
323
324static ssize_t idlecpus_store(struct device *dev,
325	struct device_attribute *attr, const char *buf, size_t count)
326{
327	unsigned long num;
328
329	if (kstrtoul(buf, 0, &num))
330		return -EINVAL;
331	mutex_lock(&isolated_cpus_lock);
332	acpi_pad_idle_cpus(num);
333	mutex_unlock(&isolated_cpus_lock);
334	return count;
335}
336
337static ssize_t idlecpus_show(struct device *dev,
338	struct device_attribute *attr, char *buf)
339{
340	return cpumap_print_to_pagebuf(false, buf,
341				       to_cpumask(pad_busy_cpus_bits));
342}
343
344static DEVICE_ATTR_RW(idlecpus);
 
 
345
346static struct attribute *acpi_pad_attrs[] = {
347	&dev_attr_idlecpus.attr,
348	&dev_attr_idlepct.attr,
349	&dev_attr_rrtime.attr,
350	NULL
351};
352
353ATTRIBUTE_GROUPS(acpi_pad);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
355/*
356 * Query firmware how many CPUs should be idle
357 * return -1 on failure
358 */
359static int acpi_pad_pur(acpi_handle handle)
360{
361	struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
362	union acpi_object *package;
363	int num = -1;
364
365	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_PUR", NULL, &buffer)))
366		return num;
367
368	if (!buffer.length || !buffer.pointer)
369		return num;
370
371	package = buffer.pointer;
372
373	if (package->type == ACPI_TYPE_PACKAGE &&
374		package->package.count == 2 &&
375		package->package.elements[0].integer.value == 1) /* rev 1 */
376
377		num = package->package.elements[1].integer.value;
378
379	kfree(buffer.pointer);
380	return num;
381}
382
383static void acpi_pad_handle_notify(acpi_handle handle)
384{
385	int num_cpus;
386	uint32_t idle_cpus;
387	struct acpi_buffer param = {
388		.length = 4,
389		.pointer = (void *)&idle_cpus,
390	};
391	u32 status;
392
393	mutex_lock(&isolated_cpus_lock);
394	num_cpus = acpi_pad_pur(handle);
395	if (num_cpus < 0) {
396		/* The ACPI specification says that if no action was performed when
397		 * processing the _PUR object, _OST should still be evaluated, albeit
398		 * with a different status code.
399		 */
400		status = ACPI_PROCESSOR_AGGREGATOR_STATUS_NO_ACTION;
401	} else {
402		status = ACPI_PROCESSOR_AGGREGATOR_STATUS_SUCCESS;
403		acpi_pad_idle_cpus(num_cpus);
404	}
405
406	idle_cpus = acpi_pad_idle_cpus_num();
407	acpi_evaluate_ost(handle, ACPI_PROCESSOR_AGGREGATOR_NOTIFY, status, &param);
408	mutex_unlock(&isolated_cpus_lock);
409}
410
411static void acpi_pad_notify(acpi_handle handle, u32 event,
412	void *data)
413{
414	struct acpi_device *adev = data;
415
416	switch (event) {
417	case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
418		acpi_pad_handle_notify(handle);
419		acpi_bus_generate_netlink_event(adev->pnp.device_class,
420			dev_name(&adev->dev), event, 0);
421		break;
422	default:
423		pr_warn("Unsupported event [0x%x]\n", event);
424		break;
425	}
426}
427
428static int acpi_pad_probe(struct platform_device *pdev)
429{
430	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
431	acpi_status status;
432
433	strscpy(acpi_device_name(adev), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
434	strscpy(acpi_device_class(adev), ACPI_PROCESSOR_AGGREGATOR_CLASS);
435
436	status = acpi_install_notify_handler(adev->handle,
437		ACPI_DEVICE_NOTIFY, acpi_pad_notify, adev);
438
439	if (ACPI_FAILURE(status))
 
 
 
440		return -ENODEV;
 
441
442	return 0;
443}
444
445static void acpi_pad_remove(struct platform_device *pdev)
446{
447	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
448
449	mutex_lock(&isolated_cpus_lock);
450	acpi_pad_idle_cpus(0);
451	mutex_unlock(&isolated_cpus_lock);
452
453	acpi_remove_notify_handler(adev->handle,
454		ACPI_DEVICE_NOTIFY, acpi_pad_notify);
 
 
455}
456
457static const struct acpi_device_id pad_device_ids[] = {
458	{"ACPI000C", 0},
459	{"", 0},
460};
461MODULE_DEVICE_TABLE(acpi, pad_device_ids);
462
463static struct platform_driver acpi_pad_driver = {
464	.probe = acpi_pad_probe,
465	.remove = acpi_pad_remove,
466	.driver = {
467		.dev_groups = acpi_pad_groups,
468		.name = "processor_aggregator",
469		.acpi_match_table = pad_device_ids,
470	},
471};
472
473static int __init acpi_pad_init(void)
474{
475	/* Xen ACPI PAD is used when running as Xen Dom0. */
476	if (xen_initial_domain())
477		return -ENODEV;
478
479	power_saving_mwait_init();
480	if (power_saving_mwait_eax == 0)
481		return -EINVAL;
482
483	return platform_driver_register(&acpi_pad_driver);
484}
485
486static void __exit acpi_pad_exit(void)
487{
488	platform_driver_unregister(&acpi_pad_driver);
489}
490
491module_init(acpi_pad_init);
492module_exit(acpi_pad_exit);
493MODULE_AUTHOR("Shaohua Li<shaohua.li@intel.com>");
494MODULE_DESCRIPTION("ACPI Processor Aggregator Driver");
495MODULE_LICENSE("GPL");

 
  1/*
  2 * acpi_pad.c ACPI Processor Aggregator Driver
  3 *
  4 * Copyright (c) 2009, Intel Corporation.
  5 *
  6 * This program is free software; you can redistribute it and/or modify it
  7 * under the terms and conditions of the GNU General Public License,
  8 * version 2, as published by the Free Software Foundation.
  9 *
 10 * This program is distributed in the hope it will be useful, but WITHOUT
 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 13 * more details.
 14 *
 15 */
 16
 17#include <linux/kernel.h>
 18#include <linux/cpumask.h>
 19#include <linux/module.h>
 20#include <linux/init.h>
 21#include <linux/types.h>
 22#include <linux/kthread.h>
 23#include <uapi/linux/sched/types.h>
 24#include <linux/freezer.h>
 25#include <linux/cpu.h>
 26#include <linux/tick.h>
 27#include <linux/slab.h>
 28#include <linux/acpi.h>
 
 
 29#include <asm/mwait.h>
 30#include <xen/xen.h>
 31
 32#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
 33#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 34#define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 
 
 
 
 35static DEFINE_MUTEX(isolated_cpus_lock);
 36static DEFINE_MUTEX(round_robin_lock);
 37
 38static unsigned long power_saving_mwait_eax;
 39
 40static unsigned char tsc_detected_unstable;
 41static unsigned char tsc_marked_unstable;
 42
 43static void power_saving_mwait_init(void)
 44{
 45	unsigned int eax, ebx, ecx, edx;
 46	unsigned int highest_cstate = 0;
 47	unsigned int highest_subcstate = 0;
 48	int i;
 49
 50	if (!boot_cpu_has(X86_FEATURE_MWAIT))
 51		return;
 52	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
 53		return;
 54
 55	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
 56
 57	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
 58	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
 59		return;
 60
 61	edx >>= MWAIT_SUBSTATE_SIZE;
 62	for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
 63		if (edx & MWAIT_SUBSTATE_MASK) {
 64			highest_cstate = i;
 65			highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
 66		}
 67	}
 68	power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
 69		(highest_subcstate - 1);
 70
 71#if defined(CONFIG_X86)
 72	switch (boot_cpu_data.x86_vendor) {
 
 73	case X86_VENDOR_AMD:
 74	case X86_VENDOR_INTEL:
 
 
 75		/*
 76		 * AMD Fam10h TSC will tick in all
 77		 * C/P/S0/S1 states when this bit is set.
 78		 */
 79		if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 80			tsc_detected_unstable = 1;
 81		break;
 82	default:
 83		/* TSC could halt in idle */
 84		tsc_detected_unstable = 1;
 85	}
 86#endif
 87}
 88
 89static unsigned long cpu_weight[NR_CPUS];
 90static int tsk_in_cpu[NR_CPUS] = {[0 ... NR_CPUS-1] = -1};
 91static DECLARE_BITMAP(pad_busy_cpus_bits, NR_CPUS);
 92static void round_robin_cpu(unsigned int tsk_index)
 93{
 94	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
 95	cpumask_var_t tmp;
 96	int cpu;
 97	unsigned long min_weight = -1;
 98	unsigned long uninitialized_var(preferred_cpu);
 99
100	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
101		return;
102
103	mutex_lock(&round_robin_lock);
104	cpumask_clear(tmp);
105	for_each_cpu(cpu, pad_busy_cpus)
106		cpumask_or(tmp, tmp, topology_sibling_cpumask(cpu));
107	cpumask_andnot(tmp, cpu_online_mask, tmp);
108	/* avoid HT sibilings if possible */
109	if (cpumask_empty(tmp))
110		cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
111	if (cpumask_empty(tmp)) {
112		mutex_unlock(&round_robin_lock);
113		free_cpumask_var(tmp);
114		return;
115	}
116	for_each_cpu(cpu, tmp) {
117		if (cpu_weight[cpu] < min_weight) {
118			min_weight = cpu_weight[cpu];
119			preferred_cpu = cpu;
120		}
121	}
122
123	if (tsk_in_cpu[tsk_index] != -1)
124		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
125	tsk_in_cpu[tsk_index] = preferred_cpu;
126	cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
127	cpu_weight[preferred_cpu]++;
128	mutex_unlock(&round_robin_lock);
129
130	set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
131
132	free_cpumask_var(tmp);
133}
134
135static void exit_round_robin(unsigned int tsk_index)
136{
137	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
138	cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
139	tsk_in_cpu[tsk_index] = -1;
 
 
 
140}
141
142static unsigned int idle_pct = 5; /* percentage */
143static unsigned int round_robin_time = 1; /* second */
144static int power_saving_thread(void *data)
145{
146	struct sched_param param = {.sched_priority = 1};
147	int do_sleep;
148	unsigned int tsk_index = (unsigned long)data;
149	u64 last_jiffies = 0;
150
151	sched_setscheduler(current, SCHED_RR, &param);
152
153	while (!kthread_should_stop()) {
154		unsigned long expire_time;
155
156		/* round robin to cpus */
157		expire_time = last_jiffies + round_robin_time * HZ;
158		if (time_before(expire_time, jiffies)) {
159			last_jiffies = jiffies;
160			round_robin_cpu(tsk_index);
161		}
162
163		do_sleep = 0;
164
165		expire_time = jiffies + HZ * (100 - idle_pct) / 100;
166
167		while (!need_resched()) {
168			if (tsc_detected_unstable && !tsc_marked_unstable) {
169				/* TSC could halt in idle, so notify users */
170				mark_tsc_unstable("TSC halts in idle");
171				tsc_marked_unstable = 1;
172			}
173			local_irq_disable();
 
 
 
174			tick_broadcast_enable();
175			tick_broadcast_enter();
176			stop_critical_timings();
177
178			mwait_idle_with_hints(power_saving_mwait_eax, 1);
179
180			start_critical_timings();
181			tick_broadcast_exit();
 
 
 
182			local_irq_enable();
183
184			if (time_before(expire_time, jiffies)) {
185				do_sleep = 1;
186				break;
187			}
188		}
189
190		/*
191		 * current sched_rt has threshold for rt task running time.
192		 * When a rt task uses 95% CPU time, the rt thread will be
193		 * scheduled out for 5% CPU time to not starve other tasks. But
194		 * the mechanism only works when all CPUs have RT task running,
195		 * as if one CPU hasn't RT task, RT task from other CPUs will
196		 * borrow CPU time from this CPU and cause RT task use > 95%
197		 * CPU time. To make 'avoid starvation' work, takes a nap here.
198		 */
199		if (unlikely(do_sleep))
200			schedule_timeout_killable(HZ * idle_pct / 100);
201
202		/* If an external event has set the need_resched flag, then
203		 * we need to deal with it, or this loop will continue to
204		 * spin without calling __mwait().
205		 */
206		if (unlikely(need_resched()))
207			schedule();
208	}
209
210	exit_round_robin(tsk_index);
211	return 0;
212}
213
214static struct task_struct *ps_tsks[NR_CPUS];
215static unsigned int ps_tsk_num;
216static int create_power_saving_task(void)
217{
218	int rc;
219
220	ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
221		(void *)(unsigned long)ps_tsk_num,
222		"acpi_pad/%d", ps_tsk_num);
223
224	if (IS_ERR(ps_tsks[ps_tsk_num])) {
225		rc = PTR_ERR(ps_tsks[ps_tsk_num]);
226		ps_tsks[ps_tsk_num] = NULL;
227	} else {
228		rc = 0;
229		ps_tsk_num++;
230	}
231
232	return rc;
233}
234
235static void destroy_power_saving_task(void)
236{
237	if (ps_tsk_num > 0) {
238		ps_tsk_num--;
239		kthread_stop(ps_tsks[ps_tsk_num]);
240		ps_tsks[ps_tsk_num] = NULL;
241	}
242}
243
244static void set_power_saving_task_num(unsigned int num)
245{
246	if (num > ps_tsk_num) {
247		while (ps_tsk_num < num) {
248			if (create_power_saving_task())
249				return;
250		}
251	} else if (num < ps_tsk_num) {
252		while (ps_tsk_num > num)
253			destroy_power_saving_task();
254	}
255}
256
257static void acpi_pad_idle_cpus(unsigned int num_cpus)
258{
259	get_online_cpus();
260
261	num_cpus = min_t(unsigned int, num_cpus, num_online_cpus());
262	set_power_saving_task_num(num_cpus);
263
264	put_online_cpus();
265}
266
267static uint32_t acpi_pad_idle_cpus_num(void)
268{
269	return ps_tsk_num;
270}
271
272static ssize_t acpi_pad_rrtime_store(struct device *dev,
273	struct device_attribute *attr, const char *buf, size_t count)
274{
275	unsigned long num;
 
276	if (kstrtoul(buf, 0, &num))
277		return -EINVAL;
278	if (num < 1 || num >= 100)
279		return -EINVAL;
280	mutex_lock(&isolated_cpus_lock);
281	round_robin_time = num;
282	mutex_unlock(&isolated_cpus_lock);
283	return count;
284}
285
286static ssize_t acpi_pad_rrtime_show(struct device *dev,
287	struct device_attribute *attr, char *buf)
288{
289	return scnprintf(buf, PAGE_SIZE, "%d\n", round_robin_time);
290}
291static DEVICE_ATTR(rrtime, S_IRUGO|S_IWUSR,
292	acpi_pad_rrtime_show,
293	acpi_pad_rrtime_store);
294
295static ssize_t acpi_pad_idlepct_store(struct device *dev,
296	struct device_attribute *attr, const char *buf, size_t count)
297{
298	unsigned long num;
 
299	if (kstrtoul(buf, 0, &num))
300		return -EINVAL;
301	if (num < 1 || num >= 100)
302		return -EINVAL;
303	mutex_lock(&isolated_cpus_lock);
304	idle_pct = num;
305	mutex_unlock(&isolated_cpus_lock);
306	return count;
307}
308
309static ssize_t acpi_pad_idlepct_show(struct device *dev,
310	struct device_attribute *attr, char *buf)
311{
312	return scnprintf(buf, PAGE_SIZE, "%d\n", idle_pct);
313}
314static DEVICE_ATTR(idlepct, S_IRUGO|S_IWUSR,
315	acpi_pad_idlepct_show,
316	acpi_pad_idlepct_store);
317
318static ssize_t acpi_pad_idlecpus_store(struct device *dev,
319	struct device_attribute *attr, const char *buf, size_t count)
320{
321	unsigned long num;
 
322	if (kstrtoul(buf, 0, &num))
323		return -EINVAL;
324	mutex_lock(&isolated_cpus_lock);
325	acpi_pad_idle_cpus(num);
326	mutex_unlock(&isolated_cpus_lock);
327	return count;
328}
329
330static ssize_t acpi_pad_idlecpus_show(struct device *dev,
331	struct device_attribute *attr, char *buf)
332{
333	return cpumap_print_to_pagebuf(false, buf,
334				       to_cpumask(pad_busy_cpus_bits));
335}
336
337static DEVICE_ATTR(idlecpus, S_IRUGO|S_IWUSR,
338	acpi_pad_idlecpus_show,
339	acpi_pad_idlecpus_store);
340
341static int acpi_pad_add_sysfs(struct acpi_device *device)
342{
343	int result;
 
 
 
344
345	result = device_create_file(&device->dev, &dev_attr_idlecpus);
346	if (result)
347		return -ENODEV;
348	result = device_create_file(&device->dev, &dev_attr_idlepct);
349	if (result) {
350		device_remove_file(&device->dev, &dev_attr_idlecpus);
351		return -ENODEV;
352	}
353	result = device_create_file(&device->dev, &dev_attr_rrtime);
354	if (result) {
355		device_remove_file(&device->dev, &dev_attr_idlecpus);
356		device_remove_file(&device->dev, &dev_attr_idlepct);
357		return -ENODEV;
358	}
359	return 0;
360}
361
362static void acpi_pad_remove_sysfs(struct acpi_device *device)
363{
364	device_remove_file(&device->dev, &dev_attr_idlecpus);
365	device_remove_file(&device->dev, &dev_attr_idlepct);
366	device_remove_file(&device->dev, &dev_attr_rrtime);
367}
368
369/*
370 * Query firmware how many CPUs should be idle
371 * return -1 on failure
372 */
373static int acpi_pad_pur(acpi_handle handle)
374{
375	struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
376	union acpi_object *package;
377	int num = -1;
378
379	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_PUR", NULL, &buffer)))
380		return num;
381
382	if (!buffer.length || !buffer.pointer)
383		return num;
384
385	package = buffer.pointer;
386
387	if (package->type == ACPI_TYPE_PACKAGE &&
388		package->package.count == 2 &&
389		package->package.elements[0].integer.value == 1) /* rev 1 */
390
391		num = package->package.elements[1].integer.value;
392
393	kfree(buffer.pointer);
394	return num;
395}
396
397static void acpi_pad_handle_notify(acpi_handle handle)
398{
399	int num_cpus;
400	uint32_t idle_cpus;
401	struct acpi_buffer param = {
402		.length = 4,
403		.pointer = (void *)&idle_cpus,
404	};
 
405
406	mutex_lock(&isolated_cpus_lock);
407	num_cpus = acpi_pad_pur(handle);
408	if (num_cpus < 0) {
409		mutex_unlock(&isolated_cpus_lock);
410		return;
 
 
 
 
 
 
411	}
412	acpi_pad_idle_cpus(num_cpus);
413	idle_cpus = acpi_pad_idle_cpus_num();
414	acpi_evaluate_ost(handle, ACPI_PROCESSOR_AGGREGATOR_NOTIFY, 0, &param);
415	mutex_unlock(&isolated_cpus_lock);
416}
417
418static void acpi_pad_notify(acpi_handle handle, u32 event,
419	void *data)
420{
421	struct acpi_device *device = data;
422
423	switch (event) {
424	case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
425		acpi_pad_handle_notify(handle);
426		acpi_bus_generate_netlink_event(device->pnp.device_class,
427			dev_name(&device->dev), event, 0);
428		break;
429	default:
430		pr_warn("Unsupported event [0x%x]\n", event);
431		break;
432	}
433}
434
435static int acpi_pad_add(struct acpi_device *device)
436{
 
437	acpi_status status;
438
439	strcpy(acpi_device_name(device), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
440	strcpy(acpi_device_class(device), ACPI_PROCESSOR_AGGREGATOR_CLASS);
441
442	if (acpi_pad_add_sysfs(device))
443		return -ENODEV;
444
445	status = acpi_install_notify_handler(device->handle,
446		ACPI_DEVICE_NOTIFY, acpi_pad_notify, device);
447	if (ACPI_FAILURE(status)) {
448		acpi_pad_remove_sysfs(device);
449		return -ENODEV;
450	}
451
452	return 0;
453}
454
455static int acpi_pad_remove(struct acpi_device *device)
456{
 
 
457	mutex_lock(&isolated_cpus_lock);
458	acpi_pad_idle_cpus(0);
459	mutex_unlock(&isolated_cpus_lock);
460
461	acpi_remove_notify_handler(device->handle,
462		ACPI_DEVICE_NOTIFY, acpi_pad_notify);
463	acpi_pad_remove_sysfs(device);
464	return 0;
465}
466
467static const struct acpi_device_id pad_device_ids[] = {
468	{"ACPI000C", 0},
469	{"", 0},
470};
471MODULE_DEVICE_TABLE(acpi, pad_device_ids);
472
473static struct acpi_driver acpi_pad_driver = {
474	.name = "processor_aggregator",
475	.class = ACPI_PROCESSOR_AGGREGATOR_CLASS,
476	.ids = pad_device_ids,
477	.ops = {
478		.add = acpi_pad_add,
479		.remove = acpi_pad_remove,
480	},
481};
482
483static int __init acpi_pad_init(void)
484{
485	/* Xen ACPI PAD is used when running as Xen Dom0. */
486	if (xen_initial_domain())
487		return -ENODEV;
488
489	power_saving_mwait_init();
490	if (power_saving_mwait_eax == 0)
491		return -EINVAL;
492
493	return acpi_bus_register_driver(&acpi_pad_driver);
494}
495
496static void __exit acpi_pad_exit(void)
497{
498	acpi_bus_unregister_driver(&acpi_pad_driver);
499}
500
501module_init(acpi_pad_init);
502module_exit(acpi_pad_exit);
503MODULE_AUTHOR("Shaohua Li<shaohua.li@intel.com>");
504MODULE_DESCRIPTION("ACPI Processor Aggregator Driver");
505MODULE_LICENSE("GPL");