Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10#include "sched.h"
11
12DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13EXPORT_SYMBOL_GPL(housekeeping_overridden);
14static cpumask_var_t housekeeping_mask;
15static unsigned int housekeeping_flags;
16
17bool housekeeping_enabled(enum hk_flags flags)
18{
19 return !!(housekeeping_flags & flags);
20}
21EXPORT_SYMBOL_GPL(housekeeping_enabled);
22
23int housekeeping_any_cpu(enum hk_flags flags)
24{
25 int cpu;
26
27 if (static_branch_unlikely(&housekeeping_overridden)) {
28 if (housekeeping_flags & flags) {
29 cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
30 if (cpu < nr_cpu_ids)
31 return cpu;
32
33 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
34 }
35 }
36 return smp_processor_id();
37}
38EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
39
40const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
41{
42 if (static_branch_unlikely(&housekeeping_overridden))
43 if (housekeeping_flags & flags)
44 return housekeeping_mask;
45 return cpu_possible_mask;
46}
47EXPORT_SYMBOL_GPL(housekeeping_cpumask);
48
49void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
50{
51 if (static_branch_unlikely(&housekeeping_overridden))
52 if (housekeeping_flags & flags)
53 set_cpus_allowed_ptr(t, housekeeping_mask);
54}
55EXPORT_SYMBOL_GPL(housekeeping_affine);
56
57bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
58{
59 if (static_branch_unlikely(&housekeeping_overridden))
60 if (housekeeping_flags & flags)
61 return cpumask_test_cpu(cpu, housekeeping_mask);
62 return true;
63}
64EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
65
66void __init housekeeping_init(void)
67{
68 if (!housekeeping_flags)
69 return;
70
71 static_branch_enable(&housekeeping_overridden);
72
73 if (housekeeping_flags & HK_FLAG_TICK)
74 sched_tick_offload_init();
75
76 /* We need at least one CPU to handle housekeeping work */
77 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
78}
79
80static int __init housekeeping_setup(char *str, enum hk_flags flags)
81{
82 cpumask_var_t non_housekeeping_mask;
83 cpumask_var_t tmp;
84 int err;
85
86 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
87 err = cpulist_parse(str, non_housekeeping_mask);
88 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
89 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
90 free_bootmem_cpumask_var(non_housekeeping_mask);
91 return 0;
92 }
93
94 alloc_bootmem_cpumask_var(&tmp);
95 if (!housekeeping_flags) {
96 alloc_bootmem_cpumask_var(&housekeeping_mask);
97 cpumask_andnot(housekeeping_mask,
98 cpu_possible_mask, non_housekeeping_mask);
99
100 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
101 if (cpumask_empty(tmp)) {
102 pr_warn("Housekeeping: must include one present CPU, "
103 "using boot CPU:%d\n", smp_processor_id());
104 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
105 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
106 }
107 } else {
108 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
109 if (cpumask_empty(tmp))
110 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
111 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
112 if (!cpumask_equal(tmp, housekeeping_mask)) {
113 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
114 free_bootmem_cpumask_var(tmp);
115 free_bootmem_cpumask_var(non_housekeeping_mask);
116 return 0;
117 }
118 }
119 free_bootmem_cpumask_var(tmp);
120
121 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
122 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
123 tick_nohz_full_setup(non_housekeeping_mask);
124 } else {
125 pr_warn("Housekeeping: nohz unsupported."
126 " Build with CONFIG_NO_HZ_FULL\n");
127 free_bootmem_cpumask_var(non_housekeeping_mask);
128 return 0;
129 }
130 }
131
132 housekeeping_flags |= flags;
133
134 free_bootmem_cpumask_var(non_housekeeping_mask);
135
136 return 1;
137}
138
139static int __init housekeeping_nohz_full_setup(char *str)
140{
141 unsigned int flags;
142
143 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
144
145 return housekeeping_setup(str, flags);
146}
147__setup("nohz_full=", housekeeping_nohz_full_setup);
148
149static int __init housekeeping_isolcpus_setup(char *str)
150{
151 unsigned int flags = 0;
152
153 while (isalpha(*str)) {
154 if (!strncmp(str, "nohz,", 5)) {
155 str += 5;
156 flags |= HK_FLAG_TICK;
157 continue;
158 }
159
160 if (!strncmp(str, "domain,", 7)) {
161 str += 7;
162 flags |= HK_FLAG_DOMAIN;
163 continue;
164 }
165
166 pr_warn("isolcpus: Error, unknown flag\n");
167 return 0;
168 }
169
170 /* Default behaviour for isolcpus without flags */
171 if (!flags)
172 flags |= HK_FLAG_DOMAIN;
173
174 return housekeeping_setup(str, flags);
175}
176__setup("isolcpus=", housekeeping_isolcpus_setup);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10#include "sched.h"
11
12DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13EXPORT_SYMBOL_GPL(housekeeping_overridden);
14static cpumask_var_t housekeeping_mask;
15static unsigned int housekeeping_flags;
16
17bool housekeeping_enabled(enum hk_flags flags)
18{
19 return !!(housekeeping_flags & flags);
20}
21EXPORT_SYMBOL_GPL(housekeeping_enabled);
22
23int housekeeping_any_cpu(enum hk_flags flags)
24{
25 int cpu;
26
27 if (static_branch_unlikely(&housekeeping_overridden)) {
28 if (housekeeping_flags & flags) {
29 cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
30 if (cpu < nr_cpu_ids)
31 return cpu;
32
33 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
34 }
35 }
36 return smp_processor_id();
37}
38EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
39
40const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
41{
42 if (static_branch_unlikely(&housekeeping_overridden))
43 if (housekeeping_flags & flags)
44 return housekeeping_mask;
45 return cpu_possible_mask;
46}
47EXPORT_SYMBOL_GPL(housekeeping_cpumask);
48
49void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
50{
51 if (static_branch_unlikely(&housekeeping_overridden))
52 if (housekeeping_flags & flags)
53 set_cpus_allowed_ptr(t, housekeeping_mask);
54}
55EXPORT_SYMBOL_GPL(housekeeping_affine);
56
57bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
58{
59 if (static_branch_unlikely(&housekeeping_overridden))
60 if (housekeeping_flags & flags)
61 return cpumask_test_cpu(cpu, housekeeping_mask);
62 return true;
63}
64EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
65
66void __init housekeeping_init(void)
67{
68 if (!housekeeping_flags)
69 return;
70
71 static_branch_enable(&housekeeping_overridden);
72
73 if (housekeeping_flags & HK_FLAG_TICK)
74 sched_tick_offload_init();
75
76 /* We need at least one CPU to handle housekeeping work */
77 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
78}
79
80static int __init housekeeping_setup(char *str, enum hk_flags flags)
81{
82 cpumask_var_t non_housekeeping_mask;
83 cpumask_var_t tmp;
84 int err;
85
86 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
87 err = cpulist_parse(str, non_housekeeping_mask);
88 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
89 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
90 free_bootmem_cpumask_var(non_housekeeping_mask);
91 return 0;
92 }
93
94 alloc_bootmem_cpumask_var(&tmp);
95 if (!housekeeping_flags) {
96 alloc_bootmem_cpumask_var(&housekeeping_mask);
97 cpumask_andnot(housekeeping_mask,
98 cpu_possible_mask, non_housekeeping_mask);
99
100 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
101 if (cpumask_empty(tmp)) {
102 pr_warn("Housekeeping: must include one present CPU, "
103 "using boot CPU:%d\n", smp_processor_id());
104 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
105 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
106 }
107 } else {
108 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
109 if (cpumask_empty(tmp))
110 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
111 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
112 if (!cpumask_equal(tmp, housekeeping_mask)) {
113 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
114 free_bootmem_cpumask_var(tmp);
115 free_bootmem_cpumask_var(non_housekeeping_mask);
116 return 0;
117 }
118 }
119 free_bootmem_cpumask_var(tmp);
120
121 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
122 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
123 tick_nohz_full_setup(non_housekeeping_mask);
124 } else {
125 pr_warn("Housekeeping: nohz unsupported."
126 " Build with CONFIG_NO_HZ_FULL\n");
127 free_bootmem_cpumask_var(non_housekeeping_mask);
128 return 0;
129 }
130 }
131
132 housekeeping_flags |= flags;
133
134 free_bootmem_cpumask_var(non_housekeeping_mask);
135
136 return 1;
137}
138
139static int __init housekeeping_nohz_full_setup(char *str)
140{
141 unsigned int flags;
142
143 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
144 HK_FLAG_MISC | HK_FLAG_KTHREAD;
145
146 return housekeeping_setup(str, flags);
147}
148__setup("nohz_full=", housekeeping_nohz_full_setup);
149
150static int __init housekeeping_isolcpus_setup(char *str)
151{
152 unsigned int flags = 0;
153 bool illegal = false;
154 char *par;
155 int len;
156
157 while (isalpha(*str)) {
158 if (!strncmp(str, "nohz,", 5)) {
159 str += 5;
160 flags |= HK_FLAG_TICK;
161 continue;
162 }
163
164 if (!strncmp(str, "domain,", 7)) {
165 str += 7;
166 flags |= HK_FLAG_DOMAIN;
167 continue;
168 }
169
170 if (!strncmp(str, "managed_irq,", 12)) {
171 str += 12;
172 flags |= HK_FLAG_MANAGED_IRQ;
173 continue;
174 }
175
176 /*
177 * Skip unknown sub-parameter and validate that it is not
178 * containing an invalid character.
179 */
180 for (par = str, len = 0; *str && *str != ','; str++, len++) {
181 if (!isalpha(*str) && *str != '_')
182 illegal = true;
183 }
184
185 if (illegal) {
186 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
187 return 0;
188 }
189
190 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
191 str++;
192 }
193
194 /* Default behaviour for isolcpus without flags */
195 if (!flags)
196 flags |= HK_FLAG_DOMAIN;
197
198 return housekeeping_setup(str, flags);
199}
200__setup("isolcpus=", housekeeping_isolcpus_setup);