1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *  Housekeeping management. Manage the targets for routine code that can run on
  4 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  5 *
  6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  8 *
  9 */
 10#include "sched.h"
 11
 12DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
 13EXPORT_SYMBOL_GPL(housekeeping_overridden);
 14static cpumask_var_t housekeeping_mask;
 15static unsigned int housekeeping_flags;
 16
 17bool housekeeping_enabled(enum hk_flags flags)
 18{
 19	return !!(housekeeping_flags & flags);
 20}
 21EXPORT_SYMBOL_GPL(housekeeping_enabled);
 22
 23int housekeeping_any_cpu(enum hk_flags flags)
 24{
 25	int cpu;
 26
 27	if (static_branch_unlikely(&housekeeping_overridden)) {
 28		if (housekeeping_flags & flags) {
 29			cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
 30			if (cpu < nr_cpu_ids)
 31				return cpu;
 32
 33			return cpumask_any_and(housekeeping_mask, cpu_online_mask);
 34		}
 35	}
 36	return smp_processor_id();
 37}
 38EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
 39
 40const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
 41{
 42	if (static_branch_unlikely(&housekeeping_overridden))
 43		if (housekeeping_flags & flags)
 44			return housekeeping_mask;
 45	return cpu_possible_mask;
 46}
 47EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 48
 49void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
 50{
 51	if (static_branch_unlikely(&housekeeping_overridden))
 52		if (housekeeping_flags & flags)
 53			set_cpus_allowed_ptr(t, housekeeping_mask);
 54}
 55EXPORT_SYMBOL_GPL(housekeeping_affine);
 56
 57bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
 58{
 59	if (static_branch_unlikely(&housekeeping_overridden))
 60		if (housekeeping_flags & flags)
 61			return cpumask_test_cpu(cpu, housekeeping_mask);
 62	return true;
 63}
 64EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
 65
 66void __init housekeeping_init(void)
 67{
 68	if (!housekeeping_flags)
 69		return;
 70
 71	static_branch_enable(&housekeeping_overridden);
 72
 73	if (housekeeping_flags & HK_FLAG_TICK)
 74		sched_tick_offload_init();
 75
 76	/* We need at least one CPU to handle housekeeping work */
 77	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
 78}
 79
 80static int __init housekeeping_setup(char *str, enum hk_flags flags)
 81{
 82	cpumask_var_t non_housekeeping_mask;
 83	cpumask_var_t tmp;
 84
 85	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
 86	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
 87		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
 88		free_bootmem_cpumask_var(non_housekeeping_mask);
 89		return 0;
 90	}
 91
 92	alloc_bootmem_cpumask_var(&tmp);
 93	if (!housekeeping_flags) {
 94		alloc_bootmem_cpumask_var(&housekeeping_mask);
 95		cpumask_andnot(housekeeping_mask,
 96			       cpu_possible_mask, non_housekeeping_mask);
 97
 98		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
 99		if (cpumask_empty(tmp)) {
100			pr_warn("Housekeeping: must include one present CPU, "
101				"using boot CPU:%d\n", smp_processor_id());
102			__cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
103			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
104		}
105	} else {
106		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
107		if (cpumask_empty(tmp))
108			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
109		cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
110		if (!cpumask_equal(tmp, housekeeping_mask)) {
111			pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
112			free_bootmem_cpumask_var(tmp);
113			free_bootmem_cpumask_var(non_housekeeping_mask);
114			return 0;
115		}
116	}
117	free_bootmem_cpumask_var(tmp);
118
119	if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
120		if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
121			tick_nohz_full_setup(non_housekeeping_mask);
122		} else {
123			pr_warn("Housekeeping: nohz unsupported."
124				" Build with CONFIG_NO_HZ_FULL\n");
125			free_bootmem_cpumask_var(non_housekeeping_mask);
126			return 0;
127		}
128	}
129
130	housekeeping_flags |= flags;
131
132	free_bootmem_cpumask_var(non_housekeeping_mask);
133
134	return 1;
135}
136
137static int __init housekeeping_nohz_full_setup(char *str)
138{
139	unsigned int flags;
140
141	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
142		HK_FLAG_MISC | HK_FLAG_KTHREAD;
143
144	return housekeeping_setup(str, flags);
145}
146__setup("nohz_full=", housekeeping_nohz_full_setup);
147
148static int __init housekeeping_isolcpus_setup(char *str)
149{
150	unsigned int flags = 0;
151	bool illegal = false;
152	char *par;
153	int len;
154
155	while (isalpha(*str)) {
156		if (!strncmp(str, "nohz,", 5)) {
157			str += 5;
158			flags |= HK_FLAG_TICK;
159			continue;
160		}
161
162		if (!strncmp(str, "domain,", 7)) {
163			str += 7;
164			flags |= HK_FLAG_DOMAIN;
165			continue;
166		}
167
168		if (!strncmp(str, "managed_irq,", 12)) {
169			str += 12;
170			flags |= HK_FLAG_MANAGED_IRQ;
171			continue;
172		}
173
174		/*
175		 * Skip unknown sub-parameter and validate that it is not
176		 * containing an invalid character.
177		 */
178		for (par = str, len = 0; *str && *str != ','; str++, len++) {
179			if (!isalpha(*str) && *str != '_')
180				illegal = true;
181		}
182
183		if (illegal) {
184			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
185			return 0;
186		}
187
188		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
189		str++;
190	}
191
192	/* Default behaviour for isolcpus without flags */
193	if (!flags)
194		flags |= HK_FLAG_DOMAIN;
195
196	return housekeeping_setup(str, flags);
197}
198__setup("isolcpus=", housekeeping_isolcpus_setup);