Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10
11enum hk_flags {
12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
13 HK_FLAG_RCU = BIT(HK_TYPE_RCU),
14 HK_FLAG_MISC = BIT(HK_TYPE_MISC),
15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED),
16 HK_FLAG_TICK = BIT(HK_TYPE_TICK),
17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
18 HK_FLAG_WQ = BIT(HK_TYPE_WQ),
19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD),
21};
22
23DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
24EXPORT_SYMBOL_GPL(housekeeping_overridden);
25
26struct housekeeping {
27 cpumask_var_t cpumasks[HK_TYPE_MAX];
28 unsigned long flags;
29};
30
31static struct housekeeping housekeeping;
32
33bool housekeeping_enabled(enum hk_type type)
34{
35 return !!(housekeeping.flags & BIT(type));
36}
37EXPORT_SYMBOL_GPL(housekeeping_enabled);
38
39int housekeeping_any_cpu(enum hk_type type)
40{
41 int cpu;
42
43 if (static_branch_unlikely(&housekeeping_overridden)) {
44 if (housekeeping.flags & BIT(type)) {
45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
46 if (cpu < nr_cpu_ids)
47 return cpu;
48
49 return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
50 }
51 }
52 return smp_processor_id();
53}
54EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
55
56const struct cpumask *housekeeping_cpumask(enum hk_type type)
57{
58 if (static_branch_unlikely(&housekeeping_overridden))
59 if (housekeeping.flags & BIT(type))
60 return housekeeping.cpumasks[type];
61 return cpu_possible_mask;
62}
63EXPORT_SYMBOL_GPL(housekeeping_cpumask);
64
65void housekeeping_affine(struct task_struct *t, enum hk_type type)
66{
67 if (static_branch_unlikely(&housekeeping_overridden))
68 if (housekeeping.flags & BIT(type))
69 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
70}
71EXPORT_SYMBOL_GPL(housekeeping_affine);
72
73bool housekeeping_test_cpu(int cpu, enum hk_type type)
74{
75 if (static_branch_unlikely(&housekeeping_overridden))
76 if (housekeeping.flags & BIT(type))
77 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
78 return true;
79}
80EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
81
82void __init housekeeping_init(void)
83{
84 enum hk_type type;
85
86 if (!housekeeping.flags)
87 return;
88
89 static_branch_enable(&housekeeping_overridden);
90
91 if (housekeeping.flags & HK_FLAG_TICK)
92 sched_tick_offload_init();
93
94 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
95 /* We need at least one CPU to handle housekeeping work */
96 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
97 }
98}
99
100static void __init housekeeping_setup_type(enum hk_type type,
101 cpumask_var_t housekeeping_staging)
102{
103
104 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
105 cpumask_copy(housekeeping.cpumasks[type],
106 housekeeping_staging);
107}
108
109static int __init housekeeping_setup(char *str, unsigned long flags)
110{
111 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
112 int err = 0;
113
114 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
115 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
116 pr_warn("Housekeeping: nohz unsupported."
117 " Build with CONFIG_NO_HZ_FULL\n");
118 return 0;
119 }
120 }
121
122 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
123 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
124 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
125 goto free_non_housekeeping_mask;
126 }
127
128 alloc_bootmem_cpumask_var(&housekeeping_staging);
129 cpumask_andnot(housekeeping_staging,
130 cpu_possible_mask, non_housekeeping_mask);
131
132 if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
133 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
134 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
135 if (!housekeeping.flags) {
136 pr_warn("Housekeeping: must include one present CPU, "
137 "using boot CPU:%d\n", smp_processor_id());
138 }
139 }
140
141 if (!housekeeping.flags) {
142 /* First setup call ("nohz_full=" or "isolcpus=") */
143 enum hk_type type;
144
145 for_each_set_bit(type, &flags, HK_TYPE_MAX)
146 housekeeping_setup_type(type, housekeeping_staging);
147 } else {
148 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
149 enum hk_type type;
150 unsigned long iter_flags = flags & housekeeping.flags;
151
152 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
153 if (!cpumask_equal(housekeeping_staging,
154 housekeeping.cpumasks[type])) {
155 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
156 goto free_housekeeping_staging;
157 }
158 }
159
160 iter_flags = flags & ~housekeeping.flags;
161
162 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
163 housekeeping_setup_type(type, housekeeping_staging);
164 }
165
166 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
167 tick_nohz_full_setup(non_housekeeping_mask);
168
169 housekeeping.flags |= flags;
170 err = 1;
171
172free_housekeeping_staging:
173 free_bootmem_cpumask_var(housekeeping_staging);
174free_non_housekeeping_mask:
175 free_bootmem_cpumask_var(non_housekeeping_mask);
176
177 return err;
178}
179
180static int __init housekeeping_nohz_full_setup(char *str)
181{
182 unsigned long flags;
183
184 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
185 HK_FLAG_MISC | HK_FLAG_KTHREAD;
186
187 return housekeeping_setup(str, flags);
188}
189__setup("nohz_full=", housekeeping_nohz_full_setup);
190
191static int __init housekeeping_isolcpus_setup(char *str)
192{
193 unsigned long flags = 0;
194 bool illegal = false;
195 char *par;
196 int len;
197
198 while (isalpha(*str)) {
199 if (!strncmp(str, "nohz,", 5)) {
200 str += 5;
201 flags |= HK_FLAG_TICK;
202 continue;
203 }
204
205 if (!strncmp(str, "domain,", 7)) {
206 str += 7;
207 flags |= HK_FLAG_DOMAIN;
208 continue;
209 }
210
211 if (!strncmp(str, "managed_irq,", 12)) {
212 str += 12;
213 flags |= HK_FLAG_MANAGED_IRQ;
214 continue;
215 }
216
217 /*
218 * Skip unknown sub-parameter and validate that it is not
219 * containing an invalid character.
220 */
221 for (par = str, len = 0; *str && *str != ','; str++, len++) {
222 if (!isalpha(*str) && *str != '_')
223 illegal = true;
224 }
225
226 if (illegal) {
227 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
228 return 0;
229 }
230
231 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
232 str++;
233 }
234
235 /* Default behaviour for isolcpus without flags */
236 if (!flags)
237 flags |= HK_FLAG_DOMAIN;
238
239 return housekeeping_setup(str, flags);
240}
241__setup("isolcpus=", housekeeping_isolcpus_setup);
1/*
2 * Housekeeping management. Manage the targets for routine code that can run on
3 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
4 *
5 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
6 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
7 *
8 */
9#include "sched.h"
10
11DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
12EXPORT_SYMBOL_GPL(housekeeping_overriden);
13static cpumask_var_t housekeeping_mask;
14static unsigned int housekeeping_flags;
15
16int housekeeping_any_cpu(enum hk_flags flags)
17{
18 if (static_branch_unlikely(&housekeeping_overriden))
19 if (housekeeping_flags & flags)
20 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
21 return smp_processor_id();
22}
23EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
24
25const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
26{
27 if (static_branch_unlikely(&housekeeping_overriden))
28 if (housekeeping_flags & flags)
29 return housekeeping_mask;
30 return cpu_possible_mask;
31}
32EXPORT_SYMBOL_GPL(housekeeping_cpumask);
33
34void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
35{
36 if (static_branch_unlikely(&housekeeping_overriden))
37 if (housekeeping_flags & flags)
38 set_cpus_allowed_ptr(t, housekeeping_mask);
39}
40EXPORT_SYMBOL_GPL(housekeeping_affine);
41
42bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
43{
44 if (static_branch_unlikely(&housekeeping_overriden))
45 if (housekeeping_flags & flags)
46 return cpumask_test_cpu(cpu, housekeeping_mask);
47 return true;
48}
49EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
50
51void __init housekeeping_init(void)
52{
53 if (!housekeeping_flags)
54 return;
55
56 static_branch_enable(&housekeeping_overriden);
57
58 if (housekeeping_flags & HK_FLAG_TICK)
59 sched_tick_offload_init();
60
61 /* We need at least one CPU to handle housekeeping work */
62 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
63}
64
65static int __init housekeeping_setup(char *str, enum hk_flags flags)
66{
67 cpumask_var_t non_housekeeping_mask;
68 int err;
69
70 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
71 err = cpulist_parse(str, non_housekeeping_mask);
72 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
73 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
74 free_bootmem_cpumask_var(non_housekeeping_mask);
75 return 0;
76 }
77
78 if (!housekeeping_flags) {
79 alloc_bootmem_cpumask_var(&housekeeping_mask);
80 cpumask_andnot(housekeeping_mask,
81 cpu_possible_mask, non_housekeeping_mask);
82 if (cpumask_empty(housekeeping_mask))
83 cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
84 } else {
85 cpumask_var_t tmp;
86
87 alloc_bootmem_cpumask_var(&tmp);
88 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
89 if (!cpumask_equal(tmp, housekeeping_mask)) {
90 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
91 free_bootmem_cpumask_var(tmp);
92 free_bootmem_cpumask_var(non_housekeeping_mask);
93 return 0;
94 }
95 free_bootmem_cpumask_var(tmp);
96 }
97
98 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
99 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
100 tick_nohz_full_setup(non_housekeeping_mask);
101 } else {
102 pr_warn("Housekeeping: nohz unsupported."
103 " Build with CONFIG_NO_HZ_FULL\n");
104 free_bootmem_cpumask_var(non_housekeeping_mask);
105 return 0;
106 }
107 }
108
109 housekeeping_flags |= flags;
110
111 free_bootmem_cpumask_var(non_housekeeping_mask);
112
113 return 1;
114}
115
116static int __init housekeeping_nohz_full_setup(char *str)
117{
118 unsigned int flags;
119
120 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
121
122 return housekeeping_setup(str, flags);
123}
124__setup("nohz_full=", housekeeping_nohz_full_setup);
125
126static int __init housekeeping_isolcpus_setup(char *str)
127{
128 unsigned int flags = 0;
129
130 while (isalpha(*str)) {
131 if (!strncmp(str, "nohz,", 5)) {
132 str += 5;
133 flags |= HK_FLAG_TICK;
134 continue;
135 }
136
137 if (!strncmp(str, "domain,", 7)) {
138 str += 7;
139 flags |= HK_FLAG_DOMAIN;
140 continue;
141 }
142
143 pr_warn("isolcpus: Error, unknown flag\n");
144 return 0;
145 }
146
147 /* Default behaviour for isolcpus without flags */
148 if (!flags)
149 flags |= HK_FLAG_DOMAIN;
150
151 return housekeeping_setup(str, flags);
152}
153__setup("isolcpus=", housekeeping_isolcpus_setup);