1/*
  2 *  Housekeeping management. Manage the targets for routine code that can run on
  3 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  4 *
  5 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  6 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  7 *
  8 */
  9#include "sched.h"
 10
 11DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
 12EXPORT_SYMBOL_GPL(housekeeping_overriden);
 13static cpumask_var_t housekeeping_mask;
 14static unsigned int housekeeping_flags;
 15
 16int housekeeping_any_cpu(enum hk_flags flags)
 17{
 18	if (static_branch_unlikely(&housekeeping_overriden))
 19		if (housekeeping_flags & flags)
 20			return cpumask_any_and(housekeeping_mask, cpu_online_mask);
 21	return smp_processor_id();
 22}
 23EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
 24
 25const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
 26{
 27	if (static_branch_unlikely(&housekeeping_overriden))
 28		if (housekeeping_flags & flags)
 29			return housekeeping_mask;
 30	return cpu_possible_mask;
 31}
 32EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 33
 34void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
 35{
 36	if (static_branch_unlikely(&housekeeping_overriden))
 37		if (housekeeping_flags & flags)
 38			set_cpus_allowed_ptr(t, housekeeping_mask);
 39}
 40EXPORT_SYMBOL_GPL(housekeeping_affine);
 41
 42bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
 43{
 44	if (static_branch_unlikely(&housekeeping_overriden))
 45		if (housekeeping_flags & flags)
 46			return cpumask_test_cpu(cpu, housekeeping_mask);
 47	return true;
 48}
 49EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
 50
 51void __init housekeeping_init(void)
 52{
 53	if (!housekeeping_flags)
 54		return;
 55
 56	static_branch_enable(&housekeeping_overriden);
 57
 58	if (housekeeping_flags & HK_FLAG_TICK)
 59		sched_tick_offload_init();
 60
 61	/* We need at least one CPU to handle housekeeping work */
 62	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
 63}
 64
 65static int __init housekeeping_setup(char *str, enum hk_flags flags)
 66{
 67	cpumask_var_t non_housekeeping_mask;
 68	int err;
 69
 70	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
 71	err = cpulist_parse(str, non_housekeeping_mask);
 72	if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
 73		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
 74		free_bootmem_cpumask_var(non_housekeeping_mask);
 75		return 0;
 76	}
 77
 78	if (!housekeeping_flags) {
 79		alloc_bootmem_cpumask_var(&housekeeping_mask);
 80		cpumask_andnot(housekeeping_mask,
 81			       cpu_possible_mask, non_housekeeping_mask);
 82		if (cpumask_empty(housekeeping_mask))
 83			cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
 84	} else {
 85		cpumask_var_t tmp;
 86
 87		alloc_bootmem_cpumask_var(&tmp);
 88		cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
 89		if (!cpumask_equal(tmp, housekeeping_mask)) {
 90			pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 91			free_bootmem_cpumask_var(tmp);
 92			free_bootmem_cpumask_var(non_housekeeping_mask);
 93			return 0;
 94		}
 95		free_bootmem_cpumask_var(tmp);
 96	}
 97
 98	if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 99		if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
100			tick_nohz_full_setup(non_housekeeping_mask);
101		} else {
102			pr_warn("Housekeeping: nohz unsupported."
103				" Build with CONFIG_NO_HZ_FULL\n");
104			free_bootmem_cpumask_var(non_housekeeping_mask);
105			return 0;
106		}
107	}
108
109	housekeeping_flags |= flags;
110
111	free_bootmem_cpumask_var(non_housekeeping_mask);
112
113	return 1;
114}
115
116static int __init housekeeping_nohz_full_setup(char *str)
117{
118	unsigned int flags;
119
120	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
121
122	return housekeeping_setup(str, flags);
123}
124__setup("nohz_full=", housekeeping_nohz_full_setup);
125
126static int __init housekeeping_isolcpus_setup(char *str)
127{
128	unsigned int flags = 0;
129
130	while (isalpha(*str)) {
131		if (!strncmp(str, "nohz,", 5)) {
132			str += 5;
133			flags |= HK_FLAG_TICK;
134			continue;
135		}
136
137		if (!strncmp(str, "domain,", 7)) {
138			str += 7;
139			flags |= HK_FLAG_DOMAIN;
140			continue;
141		}
142
143		pr_warn("isolcpus: Error, unknown flag\n");
144		return 0;
145	}
146
147	/* Default behaviour for isolcpus without flags */
148	if (!flags)
149		flags |= HK_FLAG_DOMAIN;
150
151	return housekeeping_setup(str, flags);
152}
153__setup("isolcpus=", housekeeping_isolcpus_setup);