Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2018 Linaro Limited
4 *
5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
6 *
7 * The idle injection framework provides a way to force CPUs to enter idle
8 * states for a specified fraction of time over a specified period.
9 *
10 * It relies on the smpboot kthreads feature providing common code for CPU
11 * hotplug and thread [un]parking.
12 *
13 * All of the kthreads used for idle injection are created at init time.
14 *
15 * Next, the users of the the idle injection framework provide a cpumask via
16 * its register function. The kthreads will be synchronized with respect to
17 * this cpumask.
18 *
19 * The idle + run duration is specified via separate helpers and that allows
20 * idle injection to be started.
21 *
22 * The idle injection kthreads will call play_idle_precise() with the idle
23 * duration and max allowed latency specified as per the above.
24 *
25 * After all of them have been woken up, a timer is set to start the next idle
26 * injection cycle.
27 *
28 * The timer interrupt handler will wake up the idle injection kthreads for
29 * all of the CPUs in the cpumask provided by the user.
30 *
31 * Idle injection is stopped synchronously and no leftover idle injection
32 * kthread activity after its completion is guaranteed.
33 *
34 * It is up to the user of this framework to provide a lock for higher-level
35 * synchronization to prevent race conditions like starting idle injection
36 * while unregistering from the framework.
37 */
38#define pr_fmt(fmt) "ii_dev: " fmt
39
40#include <linux/cpu.h>
41#include <linux/hrtimer.h>
42#include <linux/kthread.h>
43#include <linux/sched.h>
44#include <linux/slab.h>
45#include <linux/smpboot.h>
46
47#include <uapi/linux/sched/types.h>
48
49/**
50 * struct idle_inject_thread - task on/off switch structure
51 * @tsk: task injecting the idle cycles
52 * @should_run: whether or not to run the task (for the smpboot kthread API)
53 */
54struct idle_inject_thread {
55 struct task_struct *tsk;
56 int should_run;
57};
58
59/**
60 * struct idle_inject_device - idle injection data
61 * @timer: idle injection period timer
62 * @idle_duration_us: duration of CPU idle time to inject
63 * @run_duration_us: duration of CPU run time to allow
64 * @latency_us: max allowed latency
65 * @cpumask: mask of CPUs affected by idle injection
66 */
67struct idle_inject_device {
68 struct hrtimer timer;
69 unsigned int idle_duration_us;
70 unsigned int run_duration_us;
71 unsigned int latency_us;
72 unsigned long cpumask[];
73};
74
75static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
76static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
77
78/**
79 * idle_inject_wakeup - Wake up idle injection threads
80 * @ii_dev: target idle injection device
81 *
82 * Every idle injection task associated with the given idle injection device
83 * and running on an online CPU will be woken up.
84 */
85static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
86{
87 struct idle_inject_thread *iit;
88 unsigned int cpu;
89
90 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
91 iit = per_cpu_ptr(&idle_inject_thread, cpu);
92 iit->should_run = 1;
93 wake_up_process(iit->tsk);
94 }
95}
96
97/**
98 * idle_inject_timer_fn - idle injection timer function
99 * @timer: idle injection hrtimer
100 *
101 * This function is called when the idle injection timer expires. It wakes up
102 * idle injection tasks associated with the timer and they, in turn, invoke
103 * play_idle_precise() to inject a specified amount of CPU idle time.
104 *
105 * Return: HRTIMER_RESTART.
106 */
107static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
108{
109 unsigned int duration_us;
110 struct idle_inject_device *ii_dev =
111 container_of(timer, struct idle_inject_device, timer);
112
113 duration_us = READ_ONCE(ii_dev->run_duration_us);
114 duration_us += READ_ONCE(ii_dev->idle_duration_us);
115
116 idle_inject_wakeup(ii_dev);
117
118 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
119
120 return HRTIMER_RESTART;
121}
122
123/**
124 * idle_inject_fn - idle injection work function
125 * @cpu: the CPU owning the task
126 *
127 * This function calls play_idle_precise() to inject a specified amount of CPU
128 * idle time.
129 */
130static void idle_inject_fn(unsigned int cpu)
131{
132 struct idle_inject_device *ii_dev;
133 struct idle_inject_thread *iit;
134
135 ii_dev = per_cpu(idle_inject_device, cpu);
136 iit = per_cpu_ptr(&idle_inject_thread, cpu);
137
138 /*
139 * Let the smpboot main loop know that the task should not run again.
140 */
141 iit->should_run = 0;
142
143 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
144 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
145}
146
147/**
148 * idle_inject_set_duration - idle and run duration update helper
149 * @run_duration_us: CPU run time to allow in microseconds
150 * @idle_duration_us: CPU idle time to inject in microseconds
151 */
152void idle_inject_set_duration(struct idle_inject_device *ii_dev,
153 unsigned int run_duration_us,
154 unsigned int idle_duration_us)
155{
156 if (run_duration_us && idle_duration_us) {
157 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
158 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
159 }
160}
161
162/**
163 * idle_inject_get_duration - idle and run duration retrieval helper
164 * @run_duration_us: memory location to store the current CPU run time
165 * @idle_duration_us: memory location to store the current CPU idle time
166 */
167void idle_inject_get_duration(struct idle_inject_device *ii_dev,
168 unsigned int *run_duration_us,
169 unsigned int *idle_duration_us)
170{
171 *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
172 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
173}
174
175/**
176 * idle_inject_set_latency - set the maximum latency allowed
177 * @latency_us: set the latency requirement for the idle state
178 */
179void idle_inject_set_latency(struct idle_inject_device *ii_dev,
180 unsigned int latency_us)
181{
182 WRITE_ONCE(ii_dev->latency_us, latency_us);
183}
184
185/**
186 * idle_inject_start - start idle injections
187 * @ii_dev: idle injection control device structure
188 *
189 * The function starts idle injection by first waking up all of the idle
190 * injection kthreads associated with @ii_dev to let them inject CPU idle time
191 * sets up a timer to start the next idle injection period.
192 *
193 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
194 */
195int idle_inject_start(struct idle_inject_device *ii_dev)
196{
197 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
198 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
199
200 if (!idle_duration_us || !run_duration_us)
201 return -EINVAL;
202
203 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
204 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
205
206 idle_inject_wakeup(ii_dev);
207
208 hrtimer_start(&ii_dev->timer,
209 ns_to_ktime((idle_duration_us + run_duration_us) *
210 NSEC_PER_USEC),
211 HRTIMER_MODE_REL);
212
213 return 0;
214}
215
216/**
217 * idle_inject_stop - stops idle injections
218 * @ii_dev: idle injection control device structure
219 *
220 * The function stops idle injection and waits for the threads to finish work.
221 * If CPU idle time is being injected when this function runs, then it will
222 * wait until the end of the cycle.
223 *
224 * When it returns, there is no more idle injection kthread activity. The
225 * kthreads are scheduled out and the periodic timer is off.
226 */
227void idle_inject_stop(struct idle_inject_device *ii_dev)
228{
229 struct idle_inject_thread *iit;
230 unsigned int cpu;
231
232 pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
233 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
234
235 hrtimer_cancel(&ii_dev->timer);
236
237 /*
238 * Stopping idle injection requires all of the idle injection kthreads
239 * associated with the given cpumask to be parked and stay that way, so
240 * prevent CPUs from going online at this point. Any CPUs going online
241 * after the loop below will be covered by clearing the should_run flag
242 * that will cause the smpboot main loop to schedule them out.
243 */
244 cpu_hotplug_disable();
245
246 /*
247 * Iterate over all (online + offline) CPUs here in case one of them
248 * goes offline with the should_run flag set so as to prevent its idle
249 * injection kthread from running when the CPU goes online again after
250 * the ii_dev has been freed.
251 */
252 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
253 iit = per_cpu_ptr(&idle_inject_thread, cpu);
254 iit->should_run = 0;
255
256 wait_task_inactive(iit->tsk, 0);
257 }
258
259 cpu_hotplug_enable();
260}
261
262/**
263 * idle_inject_setup - prepare the current task for idle injection
264 * @cpu: not used
265 *
266 * Called once, this function is in charge of setting the current task's
267 * scheduler parameters to make it an RT task.
268 */
269static void idle_inject_setup(unsigned int cpu)
270{
271 sched_set_fifo(current);
272}
273
274/**
275 * idle_inject_should_run - function helper for the smpboot API
276 * @cpu: CPU the kthread is running on
277 *
278 * Return: whether or not the thread can run.
279 */
280static int idle_inject_should_run(unsigned int cpu)
281{
282 struct idle_inject_thread *iit =
283 per_cpu_ptr(&idle_inject_thread, cpu);
284
285 return iit->should_run;
286}
287
288/**
289 * idle_inject_register - initialize idle injection on a set of CPUs
290 * @cpumask: CPUs to be affected by idle injection
291 *
292 * This function creates an idle injection control device structure for the
293 * given set of CPUs and initializes the timer associated with it. It does not
294 * start any injection cycles.
295 *
296 * Return: NULL if memory allocation fails, idle injection control device
297 * pointer on success.
298 */
299struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
300{
301 struct idle_inject_device *ii_dev;
302 int cpu, cpu_rb;
303
304 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
305 if (!ii_dev)
306 return NULL;
307
308 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
309 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
310 ii_dev->timer.function = idle_inject_timer_fn;
311 ii_dev->latency_us = UINT_MAX;
312
313 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
314
315 if (per_cpu(idle_inject_device, cpu)) {
316 pr_err("cpu%d is already registered\n", cpu);
317 goto out_rollback;
318 }
319
320 per_cpu(idle_inject_device, cpu) = ii_dev;
321 }
322
323 return ii_dev;
324
325out_rollback:
326 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
327 if (cpu == cpu_rb)
328 break;
329 per_cpu(idle_inject_device, cpu_rb) = NULL;
330 }
331
332 kfree(ii_dev);
333
334 return NULL;
335}
336
337/**
338 * idle_inject_unregister - unregister idle injection control device
339 * @ii_dev: idle injection control device to unregister
340 *
341 * The function stops idle injection for the given control device,
342 * unregisters its kthreads and frees memory allocated when that device was
343 * created.
344 */
345void idle_inject_unregister(struct idle_inject_device *ii_dev)
346{
347 unsigned int cpu;
348
349 idle_inject_stop(ii_dev);
350
351 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
352 per_cpu(idle_inject_device, cpu) = NULL;
353
354 kfree(ii_dev);
355}
356
357static struct smp_hotplug_thread idle_inject_threads = {
358 .store = &idle_inject_thread.tsk,
359 .setup = idle_inject_setup,
360 .thread_fn = idle_inject_fn,
361 .thread_comm = "idle_inject/%u",
362 .thread_should_run = idle_inject_should_run,
363};
364
365static int __init idle_inject_init(void)
366{
367 return smpboot_register_percpu_thread(&idle_inject_threads);
368}
369early_initcall(idle_inject_init);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2018 Linaro Limited
4 *
5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
6 *
7 * The idle injection framework provides a way to force CPUs to enter idle
8 * states for a specified fraction of time over a specified period.
9 *
10 * It relies on the smpboot kthreads feature providing common code for CPU
11 * hotplug and thread [un]parking.
12 *
13 * All of the kthreads used for idle injection are created at init time.
14 *
15 * Next, the users of the idle injection framework provide a cpumask via
16 * its register function. The kthreads will be synchronized with respect to
17 * this cpumask.
18 *
19 * The idle + run duration is specified via separate helpers and that allows
20 * idle injection to be started.
21 *
22 * The idle injection kthreads will call play_idle_precise() with the idle
23 * duration and max allowed latency specified as per the above.
24 *
25 * After all of them have been woken up, a timer is set to start the next idle
26 * injection cycle.
27 *
28 * The timer interrupt handler will wake up the idle injection kthreads for
29 * all of the CPUs in the cpumask provided by the user.
30 *
31 * Idle injection is stopped synchronously and no leftover idle injection
32 * kthread activity after its completion is guaranteed.
33 *
34 * It is up to the user of this framework to provide a lock for higher-level
35 * synchronization to prevent race conditions like starting idle injection
36 * while unregistering from the framework.
37 */
38#define pr_fmt(fmt) "ii_dev: " fmt
39
40#include <linux/cpu.h>
41#include <linux/hrtimer.h>
42#include <linux/kthread.h>
43#include <linux/sched.h>
44#include <linux/slab.h>
45#include <linux/smpboot.h>
46#include <linux/idle_inject.h>
47
48#include <uapi/linux/sched/types.h>
49
50/**
51 * struct idle_inject_thread - task on/off switch structure
52 * @tsk: task injecting the idle cycles
53 * @should_run: whether or not to run the task (for the smpboot kthread API)
54 */
55struct idle_inject_thread {
56 struct task_struct *tsk;
57 int should_run;
58};
59
60/**
61 * struct idle_inject_device - idle injection data
62 * @timer: idle injection period timer
63 * @idle_duration_us: duration of CPU idle time to inject
64 * @run_duration_us: duration of CPU run time to allow
65 * @latency_us: max allowed latency
66 * @update: Optional callback deciding whether or not to skip idle
67 * injection in the given cycle.
68 * @cpumask: mask of CPUs affected by idle injection
69 *
70 * This structure is used to define per instance idle inject device data. Each
71 * instance has an idle duration, a run duration and mask of CPUs to inject
72 * idle.
73 *
74 * Actual CPU idle time is injected by calling kernel scheduler interface
75 * play_idle_precise(). There is one optional callback that can be registered
76 * by calling idle_inject_register_full():
77 *
78 * update() - This callback is invoked just before waking up CPUs to inject
79 * idle. If it returns false, CPUs are not woken up to inject idle in the given
80 * cycle. It also allows the caller to readjust the idle and run duration by
81 * calling idle_inject_set_duration() for the next cycle.
82 */
83struct idle_inject_device {
84 struct hrtimer timer;
85 unsigned int idle_duration_us;
86 unsigned int run_duration_us;
87 unsigned int latency_us;
88 bool (*update)(void);
89 unsigned long cpumask[];
90};
91
92static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
93static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
94
95/**
96 * idle_inject_wakeup - Wake up idle injection threads
97 * @ii_dev: target idle injection device
98 *
99 * Every idle injection task associated with the given idle injection device
100 * and running on an online CPU will be woken up.
101 */
102static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
103{
104 struct idle_inject_thread *iit;
105 unsigned int cpu;
106
107 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
108 iit = per_cpu_ptr(&idle_inject_thread, cpu);
109 iit->should_run = 1;
110 wake_up_process(iit->tsk);
111 }
112}
113
114/**
115 * idle_inject_timer_fn - idle injection timer function
116 * @timer: idle injection hrtimer
117 *
118 * This function is called when the idle injection timer expires. It wakes up
119 * idle injection tasks associated with the timer and they, in turn, invoke
120 * play_idle_precise() to inject a specified amount of CPU idle time.
121 *
122 * Return: HRTIMER_RESTART.
123 */
124static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
125{
126 unsigned int duration_us;
127 struct idle_inject_device *ii_dev =
128 container_of(timer, struct idle_inject_device, timer);
129
130 if (!ii_dev->update || (ii_dev->update && ii_dev->update()))
131 idle_inject_wakeup(ii_dev);
132
133 duration_us = READ_ONCE(ii_dev->run_duration_us);
134 duration_us += READ_ONCE(ii_dev->idle_duration_us);
135
136 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
137
138 return HRTIMER_RESTART;
139}
140
141/**
142 * idle_inject_fn - idle injection work function
143 * @cpu: the CPU owning the task
144 *
145 * This function calls play_idle_precise() to inject a specified amount of CPU
146 * idle time.
147 */
148static void idle_inject_fn(unsigned int cpu)
149{
150 struct idle_inject_device *ii_dev;
151 struct idle_inject_thread *iit;
152
153 ii_dev = per_cpu(idle_inject_device, cpu);
154 iit = per_cpu_ptr(&idle_inject_thread, cpu);
155
156 /*
157 * Let the smpboot main loop know that the task should not run again.
158 */
159 iit->should_run = 0;
160
161 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
162 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
163}
164
165/**
166 * idle_inject_set_duration - idle and run duration update helper
167 * @ii_dev: idle injection control device structure
168 * @run_duration_us: CPU run time to allow in microseconds
169 * @idle_duration_us: CPU idle time to inject in microseconds
170 */
171void idle_inject_set_duration(struct idle_inject_device *ii_dev,
172 unsigned int run_duration_us,
173 unsigned int idle_duration_us)
174{
175 if (run_duration_us + idle_duration_us) {
176 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
177 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
178 }
179 if (!run_duration_us)
180 pr_debug("CPU is forced to 100 percent idle\n");
181}
182EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);
183
184/**
185 * idle_inject_get_duration - idle and run duration retrieval helper
186 * @ii_dev: idle injection control device structure
187 * @run_duration_us: memory location to store the current CPU run time
188 * @idle_duration_us: memory location to store the current CPU idle time
189 */
190void idle_inject_get_duration(struct idle_inject_device *ii_dev,
191 unsigned int *run_duration_us,
192 unsigned int *idle_duration_us)
193{
194 *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
195 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
196}
197EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);
198
199/**
200 * idle_inject_set_latency - set the maximum latency allowed
201 * @ii_dev: idle injection control device structure
202 * @latency_us: set the latency requirement for the idle state
203 */
204void idle_inject_set_latency(struct idle_inject_device *ii_dev,
205 unsigned int latency_us)
206{
207 WRITE_ONCE(ii_dev->latency_us, latency_us);
208}
209EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);
210
211/**
212 * idle_inject_start - start idle injections
213 * @ii_dev: idle injection control device structure
214 *
215 * The function starts idle injection by first waking up all of the idle
216 * injection kthreads associated with @ii_dev to let them inject CPU idle time
217 * sets up a timer to start the next idle injection period.
218 *
219 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
220 */
221int idle_inject_start(struct idle_inject_device *ii_dev)
222{
223 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
224 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
225
226 if (!(idle_duration_us + run_duration_us))
227 return -EINVAL;
228
229 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
230 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
231
232 idle_inject_wakeup(ii_dev);
233
234 hrtimer_start(&ii_dev->timer,
235 ns_to_ktime((idle_duration_us + run_duration_us) *
236 NSEC_PER_USEC),
237 HRTIMER_MODE_REL);
238
239 return 0;
240}
241EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);
242
243/**
244 * idle_inject_stop - stops idle injections
245 * @ii_dev: idle injection control device structure
246 *
247 * The function stops idle injection and waits for the threads to finish work.
248 * If CPU idle time is being injected when this function runs, then it will
249 * wait until the end of the cycle.
250 *
251 * When it returns, there is no more idle injection kthread activity. The
252 * kthreads are scheduled out and the periodic timer is off.
253 */
254void idle_inject_stop(struct idle_inject_device *ii_dev)
255{
256 struct idle_inject_thread *iit;
257 unsigned int cpu;
258
259 pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
260 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
261
262 hrtimer_cancel(&ii_dev->timer);
263
264 /*
265 * Stopping idle injection requires all of the idle injection kthreads
266 * associated with the given cpumask to be parked and stay that way, so
267 * prevent CPUs from going online at this point. Any CPUs going online
268 * after the loop below will be covered by clearing the should_run flag
269 * that will cause the smpboot main loop to schedule them out.
270 */
271 cpu_hotplug_disable();
272
273 /*
274 * Iterate over all (online + offline) CPUs here in case one of them
275 * goes offline with the should_run flag set so as to prevent its idle
276 * injection kthread from running when the CPU goes online again after
277 * the ii_dev has been freed.
278 */
279 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
280 iit = per_cpu_ptr(&idle_inject_thread, cpu);
281 iit->should_run = 0;
282
283 wait_task_inactive(iit->tsk, TASK_ANY);
284 }
285
286 cpu_hotplug_enable();
287}
288EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);
289
290/**
291 * idle_inject_setup - prepare the current task for idle injection
292 * @cpu: not used
293 *
294 * Called once, this function is in charge of setting the current task's
295 * scheduler parameters to make it an RT task.
296 */
297static void idle_inject_setup(unsigned int cpu)
298{
299 sched_set_fifo(current);
300}
301
302/**
303 * idle_inject_should_run - function helper for the smpboot API
304 * @cpu: CPU the kthread is running on
305 *
306 * Return: whether or not the thread can run.
307 */
308static int idle_inject_should_run(unsigned int cpu)
309{
310 struct idle_inject_thread *iit =
311 per_cpu_ptr(&idle_inject_thread, cpu);
312
313 return iit->should_run;
314}
315
316/**
317 * idle_inject_register_full - initialize idle injection on a set of CPUs
318 * @cpumask: CPUs to be affected by idle injection
319 * @update: This callback is called just before waking up CPUs to inject
320 * idle
321 *
322 * This function creates an idle injection control device structure for the
323 * given set of CPUs and initializes the timer associated with it. This
324 * function also allows to register update()callback.
325 * It does not start any injection cycles.
326 *
327 * Return: NULL if memory allocation fails, idle injection control device
328 * pointer on success.
329 */
330
331struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
332 bool (*update)(void))
333{
334 struct idle_inject_device *ii_dev;
335 int cpu, cpu_rb;
336
337 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
338 if (!ii_dev)
339 return NULL;
340
341 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
342 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
343 ii_dev->timer.function = idle_inject_timer_fn;
344 ii_dev->latency_us = UINT_MAX;
345 ii_dev->update = update;
346
347 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
348
349 if (per_cpu(idle_inject_device, cpu)) {
350 pr_err("cpu%d is already registered\n", cpu);
351 goto out_rollback;
352 }
353
354 per_cpu(idle_inject_device, cpu) = ii_dev;
355 }
356
357 return ii_dev;
358
359out_rollback:
360 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
361 if (cpu == cpu_rb)
362 break;
363 per_cpu(idle_inject_device, cpu_rb) = NULL;
364 }
365
366 kfree(ii_dev);
367
368 return NULL;
369}
370EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);
371
372/**
373 * idle_inject_register - initialize idle injection on a set of CPUs
374 * @cpumask: CPUs to be affected by idle injection
375 *
376 * This function creates an idle injection control device structure for the
377 * given set of CPUs and initializes the timer associated with it. It does not
378 * start any injection cycles.
379 *
380 * Return: NULL if memory allocation fails, idle injection control device
381 * pointer on success.
382 */
383struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
384{
385 return idle_inject_register_full(cpumask, NULL);
386}
387EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);
388
389/**
390 * idle_inject_unregister - unregister idle injection control device
391 * @ii_dev: idle injection control device to unregister
392 *
393 * The function stops idle injection for the given control device,
394 * unregisters its kthreads and frees memory allocated when that device was
395 * created.
396 */
397void idle_inject_unregister(struct idle_inject_device *ii_dev)
398{
399 unsigned int cpu;
400
401 idle_inject_stop(ii_dev);
402
403 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
404 per_cpu(idle_inject_device, cpu) = NULL;
405
406 kfree(ii_dev);
407}
408EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);
409
410static struct smp_hotplug_thread idle_inject_threads = {
411 .store = &idle_inject_thread.tsk,
412 .setup = idle_inject_setup,
413 .thread_fn = idle_inject_fn,
414 .thread_comm = "idle_inject/%u",
415 .thread_should_run = idle_inject_should_run,
416};
417
418static int __init idle_inject_init(void)
419{
420 return smpboot_register_percpu_thread(&idle_inject_threads);
421}
422early_initcall(idle_inject_init);