Linux Audio

Check our new training course

Open-source upstreaming

Need help get the support for your hardware in upstream Linux?
Loading...
v4.17
  1/*
  2 * Generic helpers for smp ipi calls
  3 *
  4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
  5 */
  6
  7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8
  9#include <linux/irq_work.h>
 10#include <linux/rcupdate.h>
 11#include <linux/rculist.h>
 12#include <linux/kernel.h>
 13#include <linux/export.h>
 14#include <linux/percpu.h>
 15#include <linux/init.h>
 16#include <linux/gfp.h>
 17#include <linux/smp.h>
 18#include <linux/cpu.h>
 19#include <linux/sched.h>
 20#include <linux/sched/idle.h>
 21#include <linux/hypervisor.h>
 22
 23#include "smpboot.h"
 
 
 
 
 
 
 
 
 24
 25enum {
 26	CSD_FLAG_LOCK		= 0x01,
 27	CSD_FLAG_SYNCHRONOUS	= 0x02,
 28};
 29
 30struct call_function_data {
 31	call_single_data_t	__percpu *csd;
 
 32	cpumask_var_t		cpumask;
 33	cpumask_var_t		cpumask_ipi;
 34};
 35
 36static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
 37
 38static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
 
 
 
 39
 40static void flush_smp_call_function_queue(bool warn_cpu_offline);
 41
 42int smpcfd_prepare_cpu(unsigned int cpu)
 
 43{
 
 44	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
 45
 46	if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
 47				     cpu_to_node(cpu)))
 48		return -ENOMEM;
 49	if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
 50				     cpu_to_node(cpu))) {
 51		free_cpumask_var(cfd->cpumask);
 52		return -ENOMEM;
 53	}
 54	cfd->csd = alloc_percpu(call_single_data_t);
 55	if (!cfd->csd) {
 56		free_cpumask_var(cfd->cpumask);
 57		free_cpumask_var(cfd->cpumask_ipi);
 58		return -ENOMEM;
 59	}
 60
 61	return 0;
 62}
 63
 64int smpcfd_dead_cpu(unsigned int cpu)
 65{
 66	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
 
 
 
 67
 68	free_cpumask_var(cfd->cpumask);
 69	free_cpumask_var(cfd->cpumask_ipi);
 70	free_percpu(cfd->csd);
 71	return 0;
 72}
 73
 74int smpcfd_dying_cpu(unsigned int cpu)
 75{
 76	/*
 77	 * The IPIs for the smp-call-function callbacks queued by other
 78	 * CPUs might arrive late, either due to hardware latencies or
 79	 * because this CPU disabled interrupts (inside stop-machine)
 80	 * before the IPIs were sent. So flush out any pending callbacks
 81	 * explicitly (without waiting for the IPIs to arrive), to
 82	 * ensure that the outgoing CPU doesn't go offline with work
 83	 * still pending.
 84	 */
 85	flush_smp_call_function_queue(false);
 86	return 0;
 87}
 88
 89void __init call_function_init(void)
 90{
 
 91	int i;
 92
 93	for_each_possible_cpu(i)
 94		init_llist_head(&per_cpu(call_single_queue, i));
 
 
 
 
 95
 96	smpcfd_prepare_cpu(smp_processor_id());
 
 97}
 98
 99/*
100 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
101 *
102 * For non-synchronous ipi calls the csd can still be in use by the
103 * previous function call. For multi-cpu calls its even more interesting
104 * as we'll have to ensure no other cpu is observing our csd.
105 */
106static __always_inline void csd_lock_wait(call_single_data_t *csd)
107{
108	smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));
 
109}
110
111static __always_inline void csd_lock(call_single_data_t *csd)
112{
113	csd_lock_wait(csd);
114	csd->flags |= CSD_FLAG_LOCK;
115
116	/*
117	 * prevent CPU from reordering the above assignment
118	 * to ->flags with any subsequent assignments to other
119	 * fields of the specified call_single_data_t structure:
120	 */
121	smp_wmb();
122}
123
124static __always_inline void csd_unlock(call_single_data_t *csd)
125{
126	WARN_ON(!(csd->flags & CSD_FLAG_LOCK));
127
128	/*
129	 * ensure we're all done before releasing data:
130	 */
131	smp_store_release(&csd->flags, 0);
132}
133
134static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
 
135
136/*
137 * Insert a previously allocated call_single_data_t element
138 * for execution on the given CPU. data must already have
139 * ->func, ->info, and ->flags set.
140 */
141static int generic_exec_single(int cpu, call_single_data_t *csd,
142			       smp_call_func_t func, void *info)
143{
144	if (cpu == smp_processor_id()) {
145		unsigned long flags;
146
147		/*
148		 * We can unlock early even for the synchronous on-stack case,
149		 * since we're doing this from the same CPU..
150		 */
151		csd_unlock(csd);
152		local_irq_save(flags);
153		func(info);
154		local_irq_restore(flags);
155		return 0;
156	}
157
158
159	if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
160		csd_unlock(csd);
161		return -ENXIO;
162	}
163
164	csd->func = func;
165	csd->info = info;
 
 
166
167	/*
168	 * The list addition should be visible before sending the IPI
169	 * handler locks the list to pull the entry off it because of
170	 * normal cache coherency rules implied by spinlocks.
171	 *
172	 * If IPIs can go out of order to the cache coherency protocol
173	 * in an architecture, sufficient synchronisation should be added
174	 * to arch code to make it appear to obey cache coherency WRT
175	 * locking and barrier primitives. Generic code isn't really
176	 * equipped to do the right thing...
177	 */
178	if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
179		arch_send_call_function_single_ipi(cpu);
180
181	return 0;
182}
183
184/**
185 * generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
186 *
187 * Invoked by arch to handle an IPI for call function single.
188 * Must be called with interrupts disabled.
189 */
190void generic_smp_call_function_single_interrupt(void)
191{
192	flush_smp_call_function_queue(true);
193}
194
195/**
196 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
197 *
198 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
199 *		      offline CPU. Skip this check if set to 'false'.
200 *
201 * Flush any pending smp-call-function callbacks queued on this CPU. This is
202 * invoked by the generic IPI handler, as well as by a CPU about to go offline,
203 * to ensure that all pending IPI callbacks are run before it goes completely
204 * offline.
205 *
206 * Loop through the call_single_queue and run all the queued callbacks.
207 * Must be called with interrupts disabled.
208 */
209static void flush_smp_call_function_queue(bool warn_cpu_offline)
210{
211	struct llist_head *head;
212	struct llist_node *entry;
213	call_single_data_t *csd, *csd_next;
214	static bool warned;
215
216	lockdep_assert_irqs_disabled();
 
 
 
217
218	head = this_cpu_ptr(&call_single_queue);
219	entry = llist_del_all(head);
220	entry = llist_reverse_order(entry);
 
 
 
 
221
222	/* There shouldn't be any pending callbacks on an offline CPU. */
223	if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
224		     !warned && !llist_empty(head))) {
225		warned = true;
226		WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
 
 
227
228		/*
229		 * We don't have to use the _safe() variant here
230		 * because we are not invoking the IPI handlers yet.
 
 
 
 
 
231		 */
232		llist_for_each_entry(csd, entry, llist)
233			pr_warn("IPI callback %pS sent to offline CPU\n",
234				csd->func);
235	}
236
237	llist_for_each_entry_safe(csd, csd_next, entry, llist) {
238		smp_call_func_t func = csd->func;
239		void *info = csd->info;
240
241		/* Do we wait until *after* callback? */
242		if (csd->flags & CSD_FLAG_SYNCHRONOUS) {
243			func(info);
244			csd_unlock(csd);
245		} else {
246			csd_unlock(csd);
247			func(info);
 
 
 
 
 
 
 
 
 
248		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249	}
250
 
 
 
 
 
 
 
 
 
 
 
 
251	/*
252	 * Handle irq works queued remotely by irq_work_queue_on().
253	 * Smp functions above are typically synchronous so they
254	 * better run first since some other CPUs may be busy waiting
255	 * for them.
256	 */
257	irq_work_run();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258}
259
 
 
260/*
261 * smp_call_function_single - Run a function on a specific CPU
262 * @func: The function to run. This must be fast and non-blocking.
263 * @info: An arbitrary pointer to pass to the function.
264 * @wait: If true, wait until function has completed on other CPUs.
265 *
266 * Returns 0 on success, else a negative status code.
267 */
268int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
269			     int wait)
270{
271	call_single_data_t *csd;
272	call_single_data_t csd_stack = {
273		.flags = CSD_FLAG_LOCK | CSD_FLAG_SYNCHRONOUS,
274	};
 
275	int this_cpu;
276	int err;
277
278	/*
279	 * prevent preemption and reschedule on another processor,
280	 * as well as CPU removal
281	 */
282	this_cpu = get_cpu();
283
284	/*
285	 * Can deadlock when called with interrupts disabled.
286	 * We allow cpu's that are not yet online though, as no one else can
287	 * send smp call function interrupt to this cpu and as such deadlocks
288	 * can't happen.
289	 */
290	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
291		     && !oops_in_progress);
292
293	csd = &csd_stack;
294	if (!wait) {
295		csd = this_cpu_ptr(&csd_data);
296		csd_lock(csd);
297	}
 
 
298
299	err = generic_exec_single(cpu, csd, func, info);
 
300
301	if (wait)
302		csd_lock_wait(csd);
 
 
 
 
 
 
 
303
304	put_cpu();
305
306	return err;
307}
308EXPORT_SYMBOL(smp_call_function_single);
309
310/**
311 * smp_call_function_single_async(): Run an asynchronous function on a
312 * 			         specific CPU.
313 * @cpu: The CPU to run on.
314 * @csd: Pre-allocated and setup data structure
315 *
316 * Like smp_call_function_single(), but the call is asynchonous and
317 * can thus be done from contexts with disabled interrupts.
318 *
319 * The caller passes his own pre-allocated data structure
320 * (ie: embedded in an object) and is responsible for synchronizing it
321 * such that the IPIs performed on the @csd are strictly serialized.
322 *
323 * NOTE: Be careful, there is unfortunately no current debugging facility to
324 * validate the correctness of this serialization.
325 */
326int smp_call_function_single_async(int cpu, call_single_data_t *csd)
327{
328	int err = 0;
329
330	preempt_disable();
331
332	/* We could deadlock if we have to wait here with interrupts disabled! */
333	if (WARN_ON_ONCE(csd->flags & CSD_FLAG_LOCK))
334		csd_lock_wait(csd);
335
336	csd->flags = CSD_FLAG_LOCK;
337	smp_wmb();
338
339	err = generic_exec_single(cpu, csd, csd->func, csd->info);
340	preempt_enable();
341
342	return err;
343}
344EXPORT_SYMBOL_GPL(smp_call_function_single_async);
345
346/*
347 * smp_call_function_any - Run a function on any of the given cpus
348 * @mask: The mask of cpus it can run on.
349 * @func: The function to run. This must be fast and non-blocking.
350 * @info: An arbitrary pointer to pass to the function.
351 * @wait: If true, wait until function has completed.
352 *
353 * Returns 0 on success, else a negative status code (if no cpus were online).
 
 
354 *
355 * Selection preference:
356 *	1) current cpu if in @mask
357 *	2) any cpu of current node if in @mask
358 *	3) any other online cpu in @mask
359 */
360int smp_call_function_any(const struct cpumask *mask,
361			  smp_call_func_t func, void *info, int wait)
362{
363	unsigned int cpu;
364	const struct cpumask *nodemask;
365	int ret;
366
367	/* Try for same CPU (cheapest) */
368	cpu = get_cpu();
369	if (cpumask_test_cpu(cpu, mask))
370		goto call;
371
372	/* Try for same node. */
373	nodemask = cpumask_of_node(cpu_to_node(cpu));
374	for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
375	     cpu = cpumask_next_and(cpu, nodemask, mask)) {
376		if (cpu_online(cpu))
377			goto call;
378	}
379
380	/* Any online will do: smp_call_function_single handles nr_cpu_ids. */
381	cpu = cpumask_any_and(mask, cpu_online_mask);
382call:
383	ret = smp_call_function_single(cpu, func, info, wait);
384	put_cpu();
385	return ret;
386}
387EXPORT_SYMBOL_GPL(smp_call_function_any);
388
389/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
390 * smp_call_function_many(): Run a function on a set of other CPUs.
391 * @mask: The set of cpus to run on (only runs on online subset).
392 * @func: The function to run. This must be fast and non-blocking.
393 * @info: An arbitrary pointer to pass to the function.
394 * @wait: If true, wait (atomically) until function has completed
395 *        on other CPUs.
396 *
397 * If @wait is true, then returns once @func has returned.
398 *
399 * You must not call this function with disabled interrupts or from a
400 * hardware interrupt handler or from a bottom half handler. Preemption
401 * must be disabled when calling this function.
402 */
403void smp_call_function_many(const struct cpumask *mask,
404			    smp_call_func_t func, void *info, bool wait)
405{
406	struct call_function_data *cfd;
407	int cpu, next_cpu, this_cpu = smp_processor_id();
 
408
409	/*
410	 * Can deadlock when called with interrupts disabled.
411	 * We allow cpu's that are not yet online though, as no one else can
412	 * send smp call function interrupt to this cpu and as such deadlocks
413	 * can't happen.
414	 */
415	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
416		     && !oops_in_progress && !early_boot_irqs_disabled);
417
418	/* Try to fastpath.  So, what's a CPU they want? Ignoring this one. */
419	cpu = cpumask_first_and(mask, cpu_online_mask);
420	if (cpu == this_cpu)
421		cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
422
423	/* No online cpus?  We're done. */
424	if (cpu >= nr_cpu_ids)
425		return;
426
427	/* Do we have another CPU which isn't us? */
428	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
429	if (next_cpu == this_cpu)
430		next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
431
432	/* Fastpath: do that cpu by itself. */
433	if (next_cpu >= nr_cpu_ids) {
434		smp_call_function_single(cpu, func, info, wait);
435		return;
436	}
437
438	cfd = this_cpu_ptr(&cfd_data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
439
440	cpumask_and(cfd->cpumask, mask, cpu_online_mask);
441	__cpumask_clear_cpu(this_cpu, cfd->cpumask);
 
 
 
 
 
 
 
 
442
443	/* Some callers race with other cpus changing the passed mask */
444	if (unlikely(!cpumask_weight(cfd->cpumask)))
 
445		return;
446
447	cpumask_clear(cfd->cpumask_ipi);
448	for_each_cpu(cpu, cfd->cpumask) {
449		call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
450
451		csd_lock(csd);
452		if (wait)
453			csd->flags |= CSD_FLAG_SYNCHRONOUS;
454		csd->func = func;
455		csd->info = info;
456		if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
457			__cpumask_set_cpu(cpu, cfd->cpumask_ipi);
458	}
459
460	/* Send a message to all CPUs in the map */
461	arch_send_call_function_ipi_mask(cfd->cpumask_ipi);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
462
463	if (wait) {
464		for_each_cpu(cpu, cfd->cpumask) {
465			call_single_data_t *csd;
466
467			csd = per_cpu_ptr(cfd->csd, cpu);
468			csd_lock_wait(csd);
469		}
470	}
471}
472EXPORT_SYMBOL(smp_call_function_many);
473
474/**
475 * smp_call_function(): Run a function on all other CPUs.
476 * @func: The function to run. This must be fast and non-blocking.
477 * @info: An arbitrary pointer to pass to the function.
478 * @wait: If true, wait (atomically) until function has completed
479 *        on other CPUs.
480 *
481 * Returns 0.
482 *
483 * If @wait is true, then returns once @func has returned; otherwise
484 * it returns just before the target cpu calls @func.
485 *
486 * You must not call this function with disabled interrupts or from a
487 * hardware interrupt handler or from a bottom half handler.
488 */
489int smp_call_function(smp_call_func_t func, void *info, int wait)
490{
491	preempt_disable();
492	smp_call_function_many(cpu_online_mask, func, info, wait);
493	preempt_enable();
494
495	return 0;
496}
497EXPORT_SYMBOL(smp_call_function);
498
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
499/* Setup configured maximum number of CPUs to activate */
500unsigned int setup_max_cpus = NR_CPUS;
501EXPORT_SYMBOL(setup_max_cpus);
502
503
504/*
505 * Setup routine for controlling SMP activation
506 *
507 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
508 * activation entirely (the MPS table probe still happens, though).
509 *
510 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
511 * greater than 0, limits the maximum number of CPUs activated in
512 * SMP mode to <NUM>.
513 */
514
515void __weak arch_disable_smp_support(void) { }
516
517static int __init nosmp(char *str)
518{
519	setup_max_cpus = 0;
520	arch_disable_smp_support();
521
522	return 0;
523}
524
525early_param("nosmp", nosmp);
526
527/* this is hard limit */
528static int __init nrcpus(char *str)
529{
530	int nr_cpus;
531
532	get_option(&str, &nr_cpus);
533	if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
534		nr_cpu_ids = nr_cpus;
535
536	return 0;
537}
538
539early_param("nr_cpus", nrcpus);
540
541static int __init maxcpus(char *str)
542{
543	get_option(&str, &setup_max_cpus);
544	if (setup_max_cpus == 0)
545		arch_disable_smp_support();
546
547	return 0;
548}
549
550early_param("maxcpus", maxcpus);
551
552/* Setup number of possible processor ids */
553unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
554EXPORT_SYMBOL(nr_cpu_ids);
555
556/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
557void __init setup_nr_cpu_ids(void)
558{
559	nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
560}
561
562/* Called by boot processor to activate the rest. */
563void __init smp_init(void)
564{
565	int num_nodes, num_cpus;
566	unsigned int cpu;
567
568	idle_threads_init();
569	cpuhp_threads_init();
570
571	pr_info("Bringing up secondary CPUs ...\n");
572
573	/* FIXME: This should be done in userspace --RR */
574	for_each_present_cpu(cpu) {
575		if (num_online_cpus() >= setup_max_cpus)
576			break;
577		if (!cpu_online(cpu))
578			cpu_up(cpu);
579	}
580
581	num_nodes = num_online_nodes();
582	num_cpus  = num_online_cpus();
583	pr_info("Brought up %d node%s, %d CPU%s\n",
584		num_nodes, (num_nodes > 1 ? "s" : ""),
585		num_cpus,  (num_cpus  > 1 ? "s" : ""));
586
587	/* Any cleanup work */
 
588	smp_cpus_done(setup_max_cpus);
589}
590
591/*
592 * Call a function on all processors.  May be used during early boot while
593 * early_boot_irqs_disabled is set.  Use local_irq_save/restore() instead
594 * of local_irq_disable/enable().
595 */
596int on_each_cpu(void (*func) (void *info), void *info, int wait)
597{
598	unsigned long flags;
599	int ret = 0;
600
601	preempt_disable();
602	ret = smp_call_function(func, info, wait);
603	local_irq_save(flags);
604	func(info);
605	local_irq_restore(flags);
606	preempt_enable();
607	return ret;
608}
609EXPORT_SYMBOL(on_each_cpu);
610
611/**
612 * on_each_cpu_mask(): Run a function on processors specified by
613 * cpumask, which may include the local processor.
614 * @mask: The set of cpus to run on (only runs on online subset).
615 * @func: The function to run. This must be fast and non-blocking.
616 * @info: An arbitrary pointer to pass to the function.
617 * @wait: If true, wait (atomically) until function has completed
618 *        on other CPUs.
619 *
620 * If @wait is true, then returns once @func has returned.
621 *
622 * You must not call this function with disabled interrupts or from a
623 * hardware interrupt handler or from a bottom half handler.  The
624 * exception is that it may be used during early boot while
625 * early_boot_irqs_disabled is set.
626 */
627void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func,
628			void *info, bool wait)
629{
630	int cpu = get_cpu();
631
632	smp_call_function_many(mask, func, info, wait);
633	if (cpumask_test_cpu(cpu, mask)) {
634		unsigned long flags;
635		local_irq_save(flags);
636		func(info);
637		local_irq_restore(flags);
638	}
639	put_cpu();
640}
641EXPORT_SYMBOL(on_each_cpu_mask);
642
643/*
644 * on_each_cpu_cond(): Call a function on each processor for which
645 * the supplied function cond_func returns true, optionally waiting
646 * for all the required CPUs to finish. This may include the local
647 * processor.
648 * @cond_func:	A callback function that is passed a cpu id and
649 *		the the info parameter. The function is called
650 *		with preemption disabled. The function should
651 *		return a blooean value indicating whether to IPI
652 *		the specified CPU.
653 * @func:	The function to run on all applicable CPUs.
654 *		This must be fast and non-blocking.
655 * @info:	An arbitrary pointer to pass to both functions.
656 * @wait:	If true, wait (atomically) until function has
657 *		completed on other CPUs.
658 * @gfp_flags:	GFP flags to use when allocating the cpumask
659 *		used internally by the function.
660 *
661 * The function might sleep if the GFP flags indicates a non
662 * atomic allocation is allowed.
663 *
664 * Preemption is disabled to protect against CPUs going offline but not online.
665 * CPUs going online during the call will not be seen or sent an IPI.
666 *
667 * You must not call this function with disabled interrupts or
668 * from a hardware interrupt handler or from a bottom half handler.
669 */
670void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
671			smp_call_func_t func, void *info, bool wait,
672			gfp_t gfp_flags)
673{
674	cpumask_var_t cpus;
675	int cpu, ret;
676
677	might_sleep_if(gfpflags_allow_blocking(gfp_flags));
678
679	if (likely(zalloc_cpumask_var(&cpus, (gfp_flags|__GFP_NOWARN)))) {
680		preempt_disable();
681		for_each_online_cpu(cpu)
682			if (cond_func(cpu, info))
683				cpumask_set_cpu(cpu, cpus);
684		on_each_cpu_mask(cpus, func, info, wait);
685		preempt_enable();
686		free_cpumask_var(cpus);
687	} else {
688		/*
689		 * No free cpumask, bother. No matter, we'll
690		 * just have to IPI them one by one.
691		 */
692		preempt_disable();
693		for_each_online_cpu(cpu)
694			if (cond_func(cpu, info)) {
695				ret = smp_call_function_single(cpu, func,
696								info, wait);
697				WARN_ON_ONCE(ret);
698			}
699		preempt_enable();
700	}
701}
702EXPORT_SYMBOL(on_each_cpu_cond);
703
704static void do_nothing(void *unused)
705{
706}
707
708/**
709 * kick_all_cpus_sync - Force all cpus out of idle
710 *
711 * Used to synchronize the update of pm_idle function pointer. It's
712 * called after the pointer is updated and returns after the dummy
713 * callback function has been executed on all cpus. The execution of
714 * the function can only happen on the remote cpus after they have
715 * left the idle function which had been called via pm_idle function
716 * pointer. So it's guaranteed that nothing uses the previous pointer
717 * anymore.
718 */
719void kick_all_cpus_sync(void)
720{
721	/* Make sure the change is visible before we kick the cpus */
722	smp_mb();
723	smp_call_function(do_nothing, NULL, 1);
724}
725EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
726
727/**
728 * wake_up_all_idle_cpus - break all cpus out of idle
729 * wake_up_all_idle_cpus try to break all cpus which is in idle state even
730 * including idle polling cpus, for non-idle cpus, we will do nothing
731 * for them.
732 */
733void wake_up_all_idle_cpus(void)
734{
735	int cpu;
736
737	preempt_disable();
738	for_each_online_cpu(cpu) {
739		if (cpu == smp_processor_id())
740			continue;
741
742		wake_up_if_idle(cpu);
743	}
744	preempt_enable();
745}
746EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
747
748/**
749 * smp_call_on_cpu - Call a function on a specific cpu
750 *
751 * Used to call a function on a specific cpu and wait for it to return.
752 * Optionally make sure the call is done on a specified physical cpu via vcpu
753 * pinning in order to support virtualized environments.
754 */
755struct smp_call_on_cpu_struct {
756	struct work_struct	work;
757	struct completion	done;
758	int			(*func)(void *);
759	void			*data;
760	int			ret;
761	int			cpu;
762};
763
764static void smp_call_on_cpu_callback(struct work_struct *work)
765{
766	struct smp_call_on_cpu_struct *sscs;
767
768	sscs = container_of(work, struct smp_call_on_cpu_struct, work);
769	if (sscs->cpu >= 0)
770		hypervisor_pin_vcpu(sscs->cpu);
771	sscs->ret = sscs->func(sscs->data);
772	if (sscs->cpu >= 0)
773		hypervisor_pin_vcpu(-1);
774
775	complete(&sscs->done);
776}
777
778int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
779{
780	struct smp_call_on_cpu_struct sscs = {
781		.done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
782		.func = func,
783		.data = par,
784		.cpu  = phys ? cpu : -1,
785	};
786
787	INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
788
789	if (cpu >= nr_cpu_ids || !cpu_online(cpu))
790		return -ENXIO;
791
792	queue_work_on(cpu, system_wq, &sscs.work);
793	wait_for_completion(&sscs.done);
794
795	return sscs.ret;
796}
797EXPORT_SYMBOL_GPL(smp_call_on_cpu);
v3.1
  1/*
  2 * Generic helpers for smp ipi calls
  3 *
  4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
  5 */
 
 
 
 
  6#include <linux/rcupdate.h>
  7#include <linux/rculist.h>
  8#include <linux/kernel.h>
  9#include <linux/module.h>
 10#include <linux/percpu.h>
 11#include <linux/init.h>
 12#include <linux/gfp.h>
 13#include <linux/smp.h>
 14#include <linux/cpu.h>
 
 
 
 15
 16#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 17static struct {
 18	struct list_head	queue;
 19	raw_spinlock_t		lock;
 20} call_function __cacheline_aligned_in_smp =
 21	{
 22		.queue		= LIST_HEAD_INIT(call_function.queue),
 23		.lock		= __RAW_SPIN_LOCK_UNLOCKED(call_function.lock),
 24	};
 25
 26enum {
 27	CSD_FLAG_LOCK		= 0x01,
 
 28};
 29
 30struct call_function_data {
 31	struct call_single_data	csd;
 32	atomic_t		refs;
 33	cpumask_var_t		cpumask;
 
 34};
 35
 36static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
 37
 38struct call_single_queue {
 39	struct list_head	list;
 40	raw_spinlock_t		lock;
 41};
 42
 43static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
 44
 45static int
 46hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
 47{
 48	long cpu = (long)hcpu;
 49	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
 50
 51	switch (action) {
 52	case CPU_UP_PREPARE:
 53	case CPU_UP_PREPARE_FROZEN:
 54		if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
 55				cpu_to_node(cpu)))
 56			return notifier_from_errno(-ENOMEM);
 57		break;
 58
 59#ifdef CONFIG_HOTPLUG_CPU
 60	case CPU_UP_CANCELED:
 61	case CPU_UP_CANCELED_FROZEN:
 
 
 
 
 
 
 62
 63	case CPU_DEAD:
 64	case CPU_DEAD_FROZEN:
 65		free_cpumask_var(cfd->cpumask);
 66		break;
 67#endif
 68	};
 69
 70	return NOTIFY_OK;
 
 
 
 71}
 72
 73static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {
 74	.notifier_call		= hotplug_cfd,
 75};
 
 
 
 
 
 
 
 
 
 
 
 76
 77void __init call_function_init(void)
 78{
 79	void *cpu = (void *)(long)smp_processor_id();
 80	int i;
 81
 82	for_each_possible_cpu(i) {
 83		struct call_single_queue *q = &per_cpu(call_single_queue, i);
 84
 85		raw_spin_lock_init(&q->lock);
 86		INIT_LIST_HEAD(&q->list);
 87	}
 88
 89	hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
 90	register_cpu_notifier(&hotplug_cfd_notifier);
 91}
 92
 93/*
 94 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
 95 *
 96 * For non-synchronous ipi calls the csd can still be in use by the
 97 * previous function call. For multi-cpu calls its even more interesting
 98 * as we'll have to ensure no other cpu is observing our csd.
 99 */
100static void csd_lock_wait(struct call_single_data *data)
101{
102	while (data->flags & CSD_FLAG_LOCK)
103		cpu_relax();
104}
105
106static void csd_lock(struct call_single_data *data)
107{
108	csd_lock_wait(data);
109	data->flags = CSD_FLAG_LOCK;
110
111	/*
112	 * prevent CPU from reordering the above assignment
113	 * to ->flags with any subsequent assignments to other
114	 * fields of the specified call_single_data structure:
115	 */
116	smp_mb();
117}
118
119static void csd_unlock(struct call_single_data *data)
120{
121	WARN_ON(!(data->flags & CSD_FLAG_LOCK));
122
123	/*
124	 * ensure we're all done before releasing data:
125	 */
126	smp_mb();
 
127
128	data->flags &= ~CSD_FLAG_LOCK;
129}
130
131/*
132 * Insert a previously allocated call_single_data element
133 * for execution on the given CPU. data must already have
134 * ->func, ->info, and ->flags set.
135 */
136static
137void generic_exec_single(int cpu, struct call_single_data *data, int wait)
138{
139	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
140	unsigned long flags;
141	int ipi;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
142
143	raw_spin_lock_irqsave(&dst->lock, flags);
144	ipi = list_empty(&dst->list);
145	list_add_tail(&data->list, &dst->list);
146	raw_spin_unlock_irqrestore(&dst->lock, flags);
147
148	/*
149	 * The list addition should be visible before sending the IPI
150	 * handler locks the list to pull the entry off it because of
151	 * normal cache coherency rules implied by spinlocks.
152	 *
153	 * If IPIs can go out of order to the cache coherency protocol
154	 * in an architecture, sufficient synchronisation should be added
155	 * to arch code to make it appear to obey cache coherency WRT
156	 * locking and barrier primitives. Generic code isn't really
157	 * equipped to do the right thing...
158	 */
159	if (ipi)
160		arch_send_call_function_single_ipi(cpu);
161
162	if (wait)
163		csd_lock_wait(data);
 
 
 
 
 
 
 
 
 
 
164}
165
166/*
167 * Invoked by arch to handle an IPI for call function. Must be called with
168 * interrupts disabled.
 
 
 
 
 
 
 
 
 
 
169 */
170void generic_smp_call_function_interrupt(void)
171{
172	struct call_function_data *data;
173	int cpu = smp_processor_id();
 
 
174
175	/*
176	 * Shouldn't receive this interrupt on a cpu that is not yet online.
177	 */
178	WARN_ON_ONCE(!cpu_online(cpu));
179
180	/*
181	 * Ensure entry is visible on call_function_queue after we have
182	 * entered the IPI. See comment in smp_call_function_many.
183	 * If we don't have this, then we may miss an entry on the list
184	 * and never get another IPI to process it.
185	 */
186	smp_mb();
187
188	/*
189	 * It's ok to use list_for_each_rcu() here even though we may
190	 * delete 'pos', since list_del_rcu() doesn't clear ->next
191	 */
192	list_for_each_entry_rcu(data, &call_function.queue, csd.list) {
193		int refs;
194		smp_call_func_t func;
195
196		/*
197		 * Since we walk the list without any locks, we might
198		 * see an entry that was completed, removed from the
199		 * list and is in the process of being reused.
200		 *
201		 * We must check that the cpu is in the cpumask before
202		 * checking the refs, and both must be set before
203		 * executing the callback on this cpu.
204		 */
 
 
 
 
205
206		if (!cpumask_test_cpu(cpu, data->cpumask))
207			continue;
208
209		smp_rmb();
210
211		if (atomic_read(&data->refs) == 0)
212			continue;
213
214		func = data->csd.func;		/* save for later warn */
215		func(data->csd.info);
216
217		/*
218		 * If the cpu mask is not still set then func enabled
219		 * interrupts (BUG), and this cpu took another smp call
220		 * function interrupt and executed func(info) twice
221		 * on this cpu.  That nested execution decremented refs.
222		 */
223		if (!cpumask_test_and_clear_cpu(cpu, data->cpumask)) {
224			WARN(1, "%pf enabled interrupts and double executed\n", func);
225			continue;
226		}
227
228		refs = atomic_dec_return(&data->refs);
229		WARN_ON(refs < 0);
230
231		if (refs)
232			continue;
233
234		WARN_ON(!cpumask_empty(data->cpumask));
235
236		raw_spin_lock(&call_function.lock);
237		list_del_rcu(&data->csd.list);
238		raw_spin_unlock(&call_function.lock);
239
240		csd_unlock(&data->csd);
241	}
242
243}
244
245/*
246 * Invoked by arch to handle an IPI for call function single. Must be
247 * called from the arch with interrupts disabled.
248 */
249void generic_smp_call_function_single_interrupt(void)
250{
251	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
252	unsigned int data_flags;
253	LIST_HEAD(list);
254
255	/*
256	 * Shouldn't receive this interrupt on a cpu that is not yet online.
 
 
 
257	 */
258	WARN_ON_ONCE(!cpu_online(smp_processor_id()));
259
260	raw_spin_lock(&q->lock);
261	list_replace_init(&q->list, &list);
262	raw_spin_unlock(&q->lock);
263
264	while (!list_empty(&list)) {
265		struct call_single_data *data;
266
267		data = list_entry(list.next, struct call_single_data, list);
268		list_del(&data->list);
269
270		/*
271		 * 'data' can be invalid after this call if flags == 0
272		 * (when called through generic_exec_single()),
273		 * so save them away before making the call:
274		 */
275		data_flags = data->flags;
276
277		data->func(data->info);
278
279		/*
280		 * Unlocked CSDs are valid through generic_exec_single():
281		 */
282		if (data_flags & CSD_FLAG_LOCK)
283			csd_unlock(data);
284	}
285}
286
287static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
288
289/*
290 * smp_call_function_single - Run a function on a specific CPU
291 * @func: The function to run. This must be fast and non-blocking.
292 * @info: An arbitrary pointer to pass to the function.
293 * @wait: If true, wait until function has completed on other CPUs.
294 *
295 * Returns 0 on success, else a negative status code.
296 */
297int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
298			     int wait)
299{
300	struct call_single_data d = {
301		.flags = 0,
 
302	};
303	unsigned long flags;
304	int this_cpu;
305	int err = 0;
306
307	/*
308	 * prevent preemption and reschedule on another processor,
309	 * as well as CPU removal
310	 */
311	this_cpu = get_cpu();
312
313	/*
314	 * Can deadlock when called with interrupts disabled.
315	 * We allow cpu's that are not yet online though, as no one else can
316	 * send smp call function interrupt to this cpu and as such deadlocks
317	 * can't happen.
318	 */
319	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
320		     && !oops_in_progress);
321
322	if (cpu == this_cpu) {
323		local_irq_save(flags);
324		func(info);
325		local_irq_restore(flags);
326	} else {
327		if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
328			struct call_single_data *data = &d;
329
330			if (!wait)
331				data = &__get_cpu_var(csd_data);
332
333			csd_lock(data);
334
335			data->func = func;
336			data->info = info;
337			generic_exec_single(cpu, data, wait);
338		} else {
339			err = -ENXIO;	/* CPU not online */
340		}
341	}
342
343	put_cpu();
344
345	return err;
346}
347EXPORT_SYMBOL(smp_call_function_single);
348
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
349/*
350 * smp_call_function_any - Run a function on any of the given cpus
351 * @mask: The mask of cpus it can run on.
352 * @func: The function to run. This must be fast and non-blocking.
353 * @info: An arbitrary pointer to pass to the function.
354 * @wait: If true, wait until function has completed.
355 *
356 * Returns 0 on success, else a negative status code (if no cpus were online).
357 * Note that @wait will be implicitly turned on in case of allocation failures,
358 * since we fall back to on-stack allocation.
359 *
360 * Selection preference:
361 *	1) current cpu if in @mask
362 *	2) any cpu of current node if in @mask
363 *	3) any other online cpu in @mask
364 */
365int smp_call_function_any(const struct cpumask *mask,
366			  smp_call_func_t func, void *info, int wait)
367{
368	unsigned int cpu;
369	const struct cpumask *nodemask;
370	int ret;
371
372	/* Try for same CPU (cheapest) */
373	cpu = get_cpu();
374	if (cpumask_test_cpu(cpu, mask))
375		goto call;
376
377	/* Try for same node. */
378	nodemask = cpumask_of_node(cpu_to_node(cpu));
379	for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
380	     cpu = cpumask_next_and(cpu, nodemask, mask)) {
381		if (cpu_online(cpu))
382			goto call;
383	}
384
385	/* Any online will do: smp_call_function_single handles nr_cpu_ids. */
386	cpu = cpumask_any_and(mask, cpu_online_mask);
387call:
388	ret = smp_call_function_single(cpu, func, info, wait);
389	put_cpu();
390	return ret;
391}
392EXPORT_SYMBOL_GPL(smp_call_function_any);
393
394/**
395 * __smp_call_function_single(): Run a function on a specific CPU
396 * @cpu: The CPU to run on.
397 * @data: Pre-allocated and setup data structure
398 * @wait: If true, wait until function has completed on specified CPU.
399 *
400 * Like smp_call_function_single(), but allow caller to pass in a
401 * pre-allocated data structure. Useful for embedding @data inside
402 * other structures, for instance.
403 */
404void __smp_call_function_single(int cpu, struct call_single_data *data,
405				int wait)
406{
407	unsigned int this_cpu;
408	unsigned long flags;
409
410	this_cpu = get_cpu();
411	/*
412	 * Can deadlock when called with interrupts disabled.
413	 * We allow cpu's that are not yet online though, as no one else can
414	 * send smp call function interrupt to this cpu and as such deadlocks
415	 * can't happen.
416	 */
417	WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
418		     && !oops_in_progress);
419
420	if (cpu == this_cpu) {
421		local_irq_save(flags);
422		data->func(data->info);
423		local_irq_restore(flags);
424	} else {
425		csd_lock(data);
426		generic_exec_single(cpu, data, wait);
427	}
428	put_cpu();
429}
430
431/**
432 * smp_call_function_many(): Run a function on a set of other CPUs.
433 * @mask: The set of cpus to run on (only runs on online subset).
434 * @func: The function to run. This must be fast and non-blocking.
435 * @info: An arbitrary pointer to pass to the function.
436 * @wait: If true, wait (atomically) until function has completed
437 *        on other CPUs.
438 *
439 * If @wait is true, then returns once @func has returned.
440 *
441 * You must not call this function with disabled interrupts or from a
442 * hardware interrupt handler or from a bottom half handler. Preemption
443 * must be disabled when calling this function.
444 */
445void smp_call_function_many(const struct cpumask *mask,
446			    smp_call_func_t func, void *info, bool wait)
447{
448	struct call_function_data *data;
449	unsigned long flags;
450	int refs, cpu, next_cpu, this_cpu = smp_processor_id();
451
452	/*
453	 * Can deadlock when called with interrupts disabled.
454	 * We allow cpu's that are not yet online though, as no one else can
455	 * send smp call function interrupt to this cpu and as such deadlocks
456	 * can't happen.
457	 */
458	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
459		     && !oops_in_progress && !early_boot_irqs_disabled);
460
461	/* Try to fastpath.  So, what's a CPU they want? Ignoring this one. */
462	cpu = cpumask_first_and(mask, cpu_online_mask);
463	if (cpu == this_cpu)
464		cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
465
466	/* No online cpus?  We're done. */
467	if (cpu >= nr_cpu_ids)
468		return;
469
470	/* Do we have another CPU which isn't us? */
471	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
472	if (next_cpu == this_cpu)
473		next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
474
475	/* Fastpath: do that cpu by itself. */
476	if (next_cpu >= nr_cpu_ids) {
477		smp_call_function_single(cpu, func, info, wait);
478		return;
479	}
480
481	data = &__get_cpu_var(cfd_data);
482	csd_lock(&data->csd);
483
484	/* This BUG_ON verifies our reuse assertions and can be removed */
485	BUG_ON(atomic_read(&data->refs) || !cpumask_empty(data->cpumask));
486
487	/*
488	 * The global call function queue list add and delete are protected
489	 * by a lock, but the list is traversed without any lock, relying
490	 * on the rcu list add and delete to allow safe concurrent traversal.
491	 * We reuse the call function data without waiting for any grace
492	 * period after some other cpu removes it from the global queue.
493	 * This means a cpu might find our data block as it is being
494	 * filled out.
495	 *
496	 * We hold off the interrupt handler on the other cpu by
497	 * ordering our writes to the cpu mask vs our setting of the
498	 * refs counter.  We assert only the cpu owning the data block
499	 * will set a bit in cpumask, and each bit will only be cleared
500	 * by the subject cpu.  Each cpu must first find its bit is
501	 * set and then check that refs is set indicating the element is
502	 * ready to be processed, otherwise it must skip the entry.
503	 *
504	 * On the previous iteration refs was set to 0 by another cpu.
505	 * To avoid the use of transitivity, set the counter to 0 here
506	 * so the wmb will pair with the rmb in the interrupt handler.
507	 */
508	atomic_set(&data->refs, 0);	/* convert 3rd to 1st party write */
509
510	data->csd.func = func;
511	data->csd.info = info;
512
513	/* Ensure 0 refs is visible before mask.  Also orders func and info */
514	smp_wmb();
515
516	/* We rely on the "and" being processed before the store */
517	cpumask_and(data->cpumask, mask, cpu_online_mask);
518	cpumask_clear_cpu(this_cpu, data->cpumask);
519	refs = cpumask_weight(data->cpumask);
520
521	/* Some callers race with other cpus changing the passed mask */
522	if (unlikely(!refs)) {
523		csd_unlock(&data->csd);
524		return;
 
 
 
 
 
 
 
 
 
 
 
 
525	}
526
527	raw_spin_lock_irqsave(&call_function.lock, flags);
528	/*
529	 * Place entry at the _HEAD_ of the list, so that any cpu still
530	 * observing the entry in generic_smp_call_function_interrupt()
531	 * will not miss any other list entries:
532	 */
533	list_add_rcu(&data->csd.list, &call_function.queue);
534	/*
535	 * We rely on the wmb() in list_add_rcu to complete our writes
536	 * to the cpumask before this write to refs, which indicates
537	 * data is on the list and is ready to be processed.
538	 */
539	atomic_set(&data->refs, refs);
540	raw_spin_unlock_irqrestore(&call_function.lock, flags);
541
542	/*
543	 * Make the list addition visible before sending the ipi.
544	 * (IPIs must obey or appear to obey normal Linux cache
545	 * coherency rules -- see comment in generic_exec_single).
546	 */
547	smp_mb();
548
549	/* Send a message to all CPUs in the map */
550	arch_send_call_function_ipi_mask(data->cpumask);
 
551
552	/* Optionally wait for the CPUs to complete */
553	if (wait)
554		csd_lock_wait(&data->csd);
 
555}
556EXPORT_SYMBOL(smp_call_function_many);
557
558/**
559 * smp_call_function(): Run a function on all other CPUs.
560 * @func: The function to run. This must be fast and non-blocking.
561 * @info: An arbitrary pointer to pass to the function.
562 * @wait: If true, wait (atomically) until function has completed
563 *        on other CPUs.
564 *
565 * Returns 0.
566 *
567 * If @wait is true, then returns once @func has returned; otherwise
568 * it returns just before the target cpu calls @func.
569 *
570 * You must not call this function with disabled interrupts or from a
571 * hardware interrupt handler or from a bottom half handler.
572 */
573int smp_call_function(smp_call_func_t func, void *info, int wait)
574{
575	preempt_disable();
576	smp_call_function_many(cpu_online_mask, func, info, wait);
577	preempt_enable();
578
579	return 0;
580}
581EXPORT_SYMBOL(smp_call_function);
582
583void ipi_call_lock(void)
584{
585	raw_spin_lock(&call_function.lock);
586}
587
588void ipi_call_unlock(void)
589{
590	raw_spin_unlock(&call_function.lock);
591}
592
593void ipi_call_lock_irq(void)
594{
595	raw_spin_lock_irq(&call_function.lock);
596}
597
598void ipi_call_unlock_irq(void)
599{
600	raw_spin_unlock_irq(&call_function.lock);
601}
602#endif /* USE_GENERIC_SMP_HELPERS */
603
604/* Setup configured maximum number of CPUs to activate */
605unsigned int setup_max_cpus = NR_CPUS;
606EXPORT_SYMBOL(setup_max_cpus);
607
608
609/*
610 * Setup routine for controlling SMP activation
611 *
612 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
613 * activation entirely (the MPS table probe still happens, though).
614 *
615 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
616 * greater than 0, limits the maximum number of CPUs activated in
617 * SMP mode to <NUM>.
618 */
619
620void __weak arch_disable_smp_support(void) { }
621
622static int __init nosmp(char *str)
623{
624	setup_max_cpus = 0;
625	arch_disable_smp_support();
626
627	return 0;
628}
629
630early_param("nosmp", nosmp);
631
632/* this is hard limit */
633static int __init nrcpus(char *str)
634{
635	int nr_cpus;
636
637	get_option(&str, &nr_cpus);
638	if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
639		nr_cpu_ids = nr_cpus;
640
641	return 0;
642}
643
644early_param("nr_cpus", nrcpus);
645
646static int __init maxcpus(char *str)
647{
648	get_option(&str, &setup_max_cpus);
649	if (setup_max_cpus == 0)
650		arch_disable_smp_support();
651
652	return 0;
653}
654
655early_param("maxcpus", maxcpus);
656
657/* Setup number of possible processor ids */
658int nr_cpu_ids __read_mostly = NR_CPUS;
659EXPORT_SYMBOL(nr_cpu_ids);
660
661/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
662void __init setup_nr_cpu_ids(void)
663{
664	nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
665}
666
667/* Called by boot processor to activate the rest. */
668void __init smp_init(void)
669{
 
670	unsigned int cpu;
671
 
 
 
 
 
672	/* FIXME: This should be done in userspace --RR */
673	for_each_present_cpu(cpu) {
674		if (num_online_cpus() >= setup_max_cpus)
675			break;
676		if (!cpu_online(cpu))
677			cpu_up(cpu);
678	}
679
 
 
 
 
 
 
680	/* Any cleanup work */
681	printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
682	smp_cpus_done(setup_max_cpus);
683}
684
685/*
686 * Call a function on all processors.  May be used during early boot while
687 * early_boot_irqs_disabled is set.  Use local_irq_save/restore() instead
688 * of local_irq_disable/enable().
689 */
690int on_each_cpu(void (*func) (void *info), void *info, int wait)
691{
692	unsigned long flags;
693	int ret = 0;
694
695	preempt_disable();
696	ret = smp_call_function(func, info, wait);
697	local_irq_save(flags);
698	func(info);
699	local_irq_restore(flags);
700	preempt_enable();
701	return ret;
702}
703EXPORT_SYMBOL(on_each_cpu);