1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright 2018 Linaro Limited
  4 *
  5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
  6 *
  7 * The idle injection framework provides a way to force CPUs to enter idle
  8 * states for a specified fraction of time over a specified period.
  9 *
 10 * It relies on the smpboot kthreads feature providing common code for CPU
 11 * hotplug and thread [un]parking.
 12 *
 13 * All of the kthreads used for idle injection are created at init time.
 14 *
 15 * Next, the users of the the idle injection framework provide a cpumask via
 16 * its register function. The kthreads will be synchronized with respect to
 17 * this cpumask.
 18 *
 19 * The idle + run duration is specified via separate helpers and that allows
 20 * idle injection to be started.
 21 *
 22 * The idle injection kthreads will call play_idle() with the idle duration
 23 * specified as per the above.
 24 *
 25 * After all of them have been woken up, a timer is set to start the next idle
 26 * injection cycle.
 27 *
 28 * The timer interrupt handler will wake up the idle injection kthreads for
 29 * all of the CPUs in the cpumask provided by the user.
 30 *
 31 * Idle injection is stopped synchronously and no leftover idle injection
 32 * kthread activity after its completion is guaranteed.
 33 *
 34 * It is up to the user of this framework to provide a lock for higher-level
 35 * synchronization to prevent race conditions like starting idle injection
 36 * while unregistering from the framework.
 37 */
 38#define pr_fmt(fmt) "ii_dev: " fmt
 39
 40#include <linux/cpu.h>
 41#include <linux/hrtimer.h>
 42#include <linux/kthread.h>
 43#include <linux/sched.h>
 44#include <linux/slab.h>
 45#include <linux/smpboot.h>
 46
 47#include <uapi/linux/sched/types.h>
 48
 49/**
 50 * struct idle_inject_thread - task on/off switch structure
 51 * @tsk: task injecting the idle cycles
 52 * @should_run: whether or not to run the task (for the smpboot kthread API)
 53 */
 54struct idle_inject_thread {
 55	struct task_struct *tsk;
 56	int should_run;
 57};
 58
 59/**
 60 * struct idle_inject_device - idle injection data
 61 * @timer: idle injection period timer
 62 * @idle_duration_us: duration of CPU idle time to inject
 63 * @run_duration_us: duration of CPU run time to allow
 64 * @cpumask: mask of CPUs affected by idle injection
 65 */
 66struct idle_inject_device {
 67	struct hrtimer timer;
 68	unsigned int idle_duration_us;
 69	unsigned int run_duration_us;
 70	unsigned long int cpumask[0];
 71};
 72
 73static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
 74static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
 75
 76/**
 77 * idle_inject_wakeup - Wake up idle injection threads
 78 * @ii_dev: target idle injection device
 79 *
 80 * Every idle injection task associated with the given idle injection device
 81 * and running on an online CPU will be woken up.
 82 */
 83static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
 84{
 85	struct idle_inject_thread *iit;
 86	unsigned int cpu;
 87
 88	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
 89		iit = per_cpu_ptr(&idle_inject_thread, cpu);
 90		iit->should_run = 1;
 91		wake_up_process(iit->tsk);
 92	}
 93}
 94
 95/**
 96 * idle_inject_timer_fn - idle injection timer function
 97 * @timer: idle injection hrtimer
 98 *
 99 * This function is called when the idle injection timer expires.  It wakes up
100 * idle injection tasks associated with the timer and they, in turn, invoke
101 * play_idle() to inject a specified amount of CPU idle time.
102 *
103 * Return: HRTIMER_RESTART.
104 */
105static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
106{
107	unsigned int duration_us;
108	struct idle_inject_device *ii_dev =
109		container_of(timer, struct idle_inject_device, timer);
110
111	duration_us = READ_ONCE(ii_dev->run_duration_us);
112	duration_us += READ_ONCE(ii_dev->idle_duration_us);
113
114	idle_inject_wakeup(ii_dev);
115
116	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
117
118	return HRTIMER_RESTART;
119}
120
121/**
122 * idle_inject_fn - idle injection work function
123 * @cpu: the CPU owning the task
124 *
125 * This function calls play_idle() to inject a specified amount of CPU idle
126 * time.
127 */
128static void idle_inject_fn(unsigned int cpu)
129{
130	struct idle_inject_device *ii_dev;
131	struct idle_inject_thread *iit;
132
133	ii_dev = per_cpu(idle_inject_device, cpu);
134	iit = per_cpu_ptr(&idle_inject_thread, cpu);
135
136	/*
137	 * Let the smpboot main loop know that the task should not run again.
138	 */
139	iit->should_run = 0;
140
141	play_idle(READ_ONCE(ii_dev->idle_duration_us));
142}
143
144/**
145 * idle_inject_set_duration - idle and run duration update helper
146 * @run_duration_us: CPU run time to allow in microseconds
147 * @idle_duration_us: CPU idle time to inject in microseconds
148 */
149void idle_inject_set_duration(struct idle_inject_device *ii_dev,
150			      unsigned int run_duration_us,
151			      unsigned int idle_duration_us)
152{
153	if (run_duration_us && idle_duration_us) {
154		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
155		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
156	}
157}
158
159/**
160 * idle_inject_get_duration - idle and run duration retrieval helper
161 * @run_duration_us: memory location to store the current CPU run time
162 * @idle_duration_us: memory location to store the current CPU idle time
163 */
164void idle_inject_get_duration(struct idle_inject_device *ii_dev,
165			      unsigned int *run_duration_us,
166			      unsigned int *idle_duration_us)
167{
168	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
169	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
170}
171
172/**
173 * idle_inject_start - start idle injections
174 * @ii_dev: idle injection control device structure
175 *
176 * The function starts idle injection by first waking up all of the idle
177 * injection kthreads associated with @ii_dev to let them inject CPU idle time
178 * sets up a timer to start the next idle injection period.
179 *
180 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
181 */
182int idle_inject_start(struct idle_inject_device *ii_dev)
183{
184	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
185	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
186
187	if (!idle_duration_us || !run_duration_us)
188		return -EINVAL;
189
190	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
191		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
192
193	idle_inject_wakeup(ii_dev);
194
195	hrtimer_start(&ii_dev->timer,
196		      ns_to_ktime((idle_duration_us + run_duration_us) *
197				  NSEC_PER_USEC),
198		      HRTIMER_MODE_REL);
199
200	return 0;
201}
202
203/**
204 * idle_inject_stop - stops idle injections
205 * @ii_dev: idle injection control device structure
206 *
207 * The function stops idle injection and waits for the threads to finish work.
208 * If CPU idle time is being injected when this function runs, then it will
209 * wait until the end of the cycle.
210 *
211 * When it returns, there is no more idle injection kthread activity.  The
212 * kthreads are scheduled out and the periodic timer is off.
213 */
214void idle_inject_stop(struct idle_inject_device *ii_dev)
215{
216	struct idle_inject_thread *iit;
217	unsigned int cpu;
218
219	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
220		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
221
222	hrtimer_cancel(&ii_dev->timer);
223
224	/*
225	 * Stopping idle injection requires all of the idle injection kthreads
226	 * associated with the given cpumask to be parked and stay that way, so
227	 * prevent CPUs from going online at this point.  Any CPUs going online
228	 * after the loop below will be covered by clearing the should_run flag
229	 * that will cause the smpboot main loop to schedule them out.
230	 */
231	cpu_hotplug_disable();
232
233	/*
234	 * Iterate over all (online + offline) CPUs here in case one of them
235	 * goes offline with the should_run flag set so as to prevent its idle
236	 * injection kthread from running when the CPU goes online again after
237	 * the ii_dev has been freed.
238	 */
239	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
240		iit = per_cpu_ptr(&idle_inject_thread, cpu);
241		iit->should_run = 0;
242
243		wait_task_inactive(iit->tsk, 0);
244	}
245
246	cpu_hotplug_enable();
247}
248
249/**
250 * idle_inject_setup - prepare the current task for idle injection
251 * @cpu: not used
252 *
253 * Called once, this function is in charge of setting the current task's
254 * scheduler parameters to make it an RT task.
255 */
256static void idle_inject_setup(unsigned int cpu)
257{
258	struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 };
259
260	sched_setscheduler(current, SCHED_FIFO, &param);
261}
262
263/**
264 * idle_inject_should_run - function helper for the smpboot API
265 * @cpu: CPU the kthread is running on
266 *
267 * Return: whether or not the thread can run.
268 */
269static int idle_inject_should_run(unsigned int cpu)
270{
271	struct idle_inject_thread *iit =
272		per_cpu_ptr(&idle_inject_thread, cpu);
273
274	return iit->should_run;
275}
276
277/**
278 * idle_inject_register - initialize idle injection on a set of CPUs
279 * @cpumask: CPUs to be affected by idle injection
280 *
281 * This function creates an idle injection control device structure for the
282 * given set of CPUs and initializes the timer associated with it.  It does not
283 * start any injection cycles.
284 *
285 * Return: NULL if memory allocation fails, idle injection control device
286 * pointer on success.
287 */
288struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
289{
290	struct idle_inject_device *ii_dev;
291	int cpu, cpu_rb;
292
293	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
294	if (!ii_dev)
295		return NULL;
296
297	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
298	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
299	ii_dev->timer.function = idle_inject_timer_fn;
300
301	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
302
303		if (per_cpu(idle_inject_device, cpu)) {
304			pr_err("cpu%d is already registered\n", cpu);
305			goto out_rollback;
306		}
307
308		per_cpu(idle_inject_device, cpu) = ii_dev;
309	}
310
311	return ii_dev;
312
313out_rollback:
314	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
315		if (cpu == cpu_rb)
316			break;
317		per_cpu(idle_inject_device, cpu_rb) = NULL;
318	}
319
320	kfree(ii_dev);
321
322	return NULL;
323}
324
325/**
326 * idle_inject_unregister - unregister idle injection control device
327 * @ii_dev: idle injection control device to unregister
328 *
329 * The function stops idle injection for the given control device,
330 * unregisters its kthreads and frees memory allocated when that device was
331 * created.
332 */
333void idle_inject_unregister(struct idle_inject_device *ii_dev)
334{
335	unsigned int cpu;
336
337	idle_inject_stop(ii_dev);
338
339	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
340		per_cpu(idle_inject_device, cpu) = NULL;
341
342	kfree(ii_dev);
343}
344
345static struct smp_hotplug_thread idle_inject_threads = {
346	.store = &idle_inject_thread.tsk,
347	.setup = idle_inject_setup,
348	.thread_fn = idle_inject_fn,
349	.thread_comm = "idle_inject/%u",
350	.thread_should_run = idle_inject_should_run,
351};
352
353static int __init idle_inject_init(void)
354{
355	return smpboot_register_percpu_thread(&idle_inject_threads);
356}
357early_initcall(idle_inject_init);