1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *  Housekeeping management. Manage the targets for routine code that can run on
  4 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  5 *
  6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  8 *
  9 */
 10
 11enum hk_flags {
 12	HK_FLAG_TIMER		= BIT(HK_TYPE_TIMER),
 13	HK_FLAG_RCU		= BIT(HK_TYPE_RCU),
 14	HK_FLAG_MISC		= BIT(HK_TYPE_MISC),
 15	HK_FLAG_SCHED		= BIT(HK_TYPE_SCHED),
 16	HK_FLAG_TICK		= BIT(HK_TYPE_TICK),
 17	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
 18	HK_FLAG_WQ		= BIT(HK_TYPE_WQ),
 19	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
 20	HK_FLAG_KTHREAD		= BIT(HK_TYPE_KTHREAD),
 21};
 22
 23DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
 24EXPORT_SYMBOL_GPL(housekeeping_overridden);
 25
 26struct housekeeping {
 27	cpumask_var_t cpumasks[HK_TYPE_MAX];
 28	unsigned long flags;
 29};
 30
 31static struct housekeeping housekeeping;
 32
 33bool housekeeping_enabled(enum hk_type type)
 34{
 35	return !!(housekeeping.flags & BIT(type));
 36}
 37EXPORT_SYMBOL_GPL(housekeeping_enabled);
 38
 39int housekeeping_any_cpu(enum hk_type type)
 40{
 41	int cpu;
 42
 43	if (static_branch_unlikely(&housekeeping_overridden)) {
 44		if (housekeeping.flags & BIT(type)) {
 45			cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
 46			if (cpu < nr_cpu_ids)
 47				return cpu;
 48
 49			return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
 50		}
 51	}
 52	return smp_processor_id();
 53}
 54EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
 55
 56const struct cpumask *housekeeping_cpumask(enum hk_type type)
 57{
 58	if (static_branch_unlikely(&housekeeping_overridden))
 59		if (housekeeping.flags & BIT(type))
 60			return housekeeping.cpumasks[type];
 61	return cpu_possible_mask;
 62}
 63EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 64
 65void housekeeping_affine(struct task_struct *t, enum hk_type type)
 66{
 67	if (static_branch_unlikely(&housekeeping_overridden))
 68		if (housekeeping.flags & BIT(type))
 69			set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
 70}
 71EXPORT_SYMBOL_GPL(housekeeping_affine);
 72
 73bool housekeeping_test_cpu(int cpu, enum hk_type type)
 74{
 75	if (static_branch_unlikely(&housekeeping_overridden))
 76		if (housekeeping.flags & BIT(type))
 77			return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
 78	return true;
 79}
 80EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
 81
 82void __init housekeeping_init(void)
 83{
 84	enum hk_type type;
 85
 86	if (!housekeeping.flags)
 87		return;
 88
 89	static_branch_enable(&housekeeping_overridden);
 90
 91	if (housekeeping.flags & HK_FLAG_TICK)
 92		sched_tick_offload_init();
 93
 94	for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
 95		/* We need at least one CPU to handle housekeeping work */
 96		WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
 97	}
 98}
 99
100static void __init housekeeping_setup_type(enum hk_type type,
101					   cpumask_var_t housekeeping_staging)
102{
103
104	alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
105	cpumask_copy(housekeeping.cpumasks[type],
106		     housekeeping_staging);
107}
108
109static int __init housekeeping_setup(char *str, unsigned long flags)
110{
111	cpumask_var_t non_housekeeping_mask, housekeeping_staging;
112	int err = 0;
113
114	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
115		if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
116			pr_warn("Housekeeping: nohz unsupported."
117				" Build with CONFIG_NO_HZ_FULL\n");
118			return 0;
119		}
120	}
121
122	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
123	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
124		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
125		goto free_non_housekeeping_mask;
126	}
127
128	alloc_bootmem_cpumask_var(&housekeeping_staging);
129	cpumask_andnot(housekeeping_staging,
130		       cpu_possible_mask, non_housekeeping_mask);
131
132	if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
133		__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
134		__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
135		if (!housekeeping.flags) {
136			pr_warn("Housekeeping: must include one present CPU, "
137				"using boot CPU:%d\n", smp_processor_id());
138		}
139	}
140
141	if (!housekeeping.flags) {
142		/* First setup call ("nohz_full=" or "isolcpus=") */
143		enum hk_type type;
144
145		for_each_set_bit(type, &flags, HK_TYPE_MAX)
146			housekeeping_setup_type(type, housekeeping_staging);
147	} else {
148		/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
149		enum hk_type type;
150		unsigned long iter_flags = flags & housekeeping.flags;
151
152		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
153			if (!cpumask_equal(housekeeping_staging,
154					   housekeeping.cpumasks[type])) {
155				pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
156				goto free_housekeeping_staging;
157			}
158		}
159
160		iter_flags = flags & ~housekeeping.flags;
161
162		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
163			housekeeping_setup_type(type, housekeeping_staging);
164	}
165
166	if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
167		tick_nohz_full_setup(non_housekeeping_mask);
168
169	housekeeping.flags |= flags;
170	err = 1;
171
172free_housekeeping_staging:
173	free_bootmem_cpumask_var(housekeeping_staging);
174free_non_housekeeping_mask:
175	free_bootmem_cpumask_var(non_housekeeping_mask);
176
177	return err;
178}
179
180static int __init housekeeping_nohz_full_setup(char *str)
181{
182	unsigned long flags;
183
184	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
185		HK_FLAG_MISC | HK_FLAG_KTHREAD;
186
187	return housekeeping_setup(str, flags);
188}
189__setup("nohz_full=", housekeeping_nohz_full_setup);
190
191static int __init housekeeping_isolcpus_setup(char *str)
192{
193	unsigned long flags = 0;
194	bool illegal = false;
195	char *par;
196	int len;
197
198	while (isalpha(*str)) {
199		if (!strncmp(str, "nohz,", 5)) {
200			str += 5;
201			flags |= HK_FLAG_TICK;
202			continue;
203		}
204
205		if (!strncmp(str, "domain,", 7)) {
206			str += 7;
207			flags |= HK_FLAG_DOMAIN;
208			continue;
209		}
210
211		if (!strncmp(str, "managed_irq,", 12)) {
212			str += 12;
213			flags |= HK_FLAG_MANAGED_IRQ;
214			continue;
215		}
216
217		/*
218		 * Skip unknown sub-parameter and validate that it is not
219		 * containing an invalid character.
220		 */
221		for (par = str, len = 0; *str && *str != ','; str++, len++) {
222			if (!isalpha(*str) && *str != '_')
223				illegal = true;
224		}
225
226		if (illegal) {
227			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
228			return 0;
229		}
230
231		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
232		str++;
233	}
234
235	/* Default behaviour for isolcpus without flags */
236	if (!flags)
237		flags |= HK_FLAG_DOMAIN;
238
239	return housekeeping_setup(str, flags);
240}
241__setup("isolcpus=", housekeeping_isolcpus_setup);