1/* CPU control.
   2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3 *
   4 * This code is licenced under the GPL.
   5 */
   6#include <linux/sched/mm.h>
   7#include <linux/proc_fs.h>
   8#include <linux/smp.h>
   9#include <linux/init.h>
  10#include <linux/notifier.h>
  11#include <linux/sched/signal.h>
  12#include <linux/sched/hotplug.h>
  13#include <linux/sched/isolation.h>
  14#include <linux/sched/task.h>
  15#include <linux/sched/smt.h>
  16#include <linux/unistd.h>
  17#include <linux/cpu.h>
  18#include <linux/oom.h>
  19#include <linux/rcupdate.h>
  20#include <linux/export.h>
  21#include <linux/bug.h>
  22#include <linux/kthread.h>
  23#include <linux/stop_machine.h>
  24#include <linux/mutex.h>
  25#include <linux/gfp.h>
  26#include <linux/suspend.h>
  27#include <linux/lockdep.h>
  28#include <linux/tick.h>
  29#include <linux/irq.h>
  30#include <linux/nmi.h>
  31#include <linux/smpboot.h>
  32#include <linux/relay.h>
  33#include <linux/slab.h>
  34#include <linux/percpu-rwsem.h>
 
  35
  36#include <trace/events/power.h>
  37#define CREATE_TRACE_POINTS
  38#include <trace/events/cpuhp.h>
  39
  40#include "smpboot.h"
  41
  42/**
  43 * cpuhp_cpu_state - Per cpu hotplug state storage
  44 * @state:	The current cpu state
  45 * @target:	The target state
  46 * @thread:	Pointer to the hotplug thread
  47 * @should_run:	Thread should execute
  48 * @rollback:	Perform a rollback
  49 * @single:	Single callback invocation
  50 * @bringup:	Single callback bringup or teardown selector
  51 * @cb_state:	The state for a single callback (install/uninstall)
  52 * @result:	Result of the operation
  53 * @done_up:	Signal completion to the issuer of the task for cpu-up
  54 * @done_down:	Signal completion to the issuer of the task for cpu-down
  55 */
  56struct cpuhp_cpu_state {
  57	enum cpuhp_state	state;
  58	enum cpuhp_state	target;
  59	enum cpuhp_state	fail;
  60#ifdef CONFIG_SMP
  61	struct task_struct	*thread;
  62	bool			should_run;
  63	bool			rollback;
  64	bool			single;
  65	bool			bringup;
 
  66	struct hlist_node	*node;
  67	struct hlist_node	*last;
  68	enum cpuhp_state	cb_state;
  69	int			result;
  70	struct completion	done_up;
  71	struct completion	done_down;
  72#endif
  73};
  74
  75static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  76	.fail = CPUHP_INVALID,
  77};
  78
  79#ifdef CONFIG_SMP
  80cpumask_t cpus_booted_once_mask;
  81#endif
  82
  83#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  84static struct lockdep_map cpuhp_state_up_map =
  85	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  86static struct lockdep_map cpuhp_state_down_map =
  87	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  88
  89
  90static inline void cpuhp_lock_acquire(bool bringup)
  91{
  92	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  93}
  94
  95static inline void cpuhp_lock_release(bool bringup)
  96{
  97	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  98}
  99#else
 100
 101static inline void cpuhp_lock_acquire(bool bringup) { }
 102static inline void cpuhp_lock_release(bool bringup) { }
 103
 104#endif
 105
 106/**
 107 * cpuhp_step - Hotplug state machine step
 108 * @name:	Name of the step
 109 * @startup:	Startup function of the step
 110 * @teardown:	Teardown function of the step
 111 * @cant_stop:	Bringup/teardown can't be stopped at this step
 112 */
 113struct cpuhp_step {
 114	const char		*name;
 115	union {
 116		int		(*single)(unsigned int cpu);
 117		int		(*multi)(unsigned int cpu,
 118					 struct hlist_node *node);
 119	} startup;
 120	union {
 121		int		(*single)(unsigned int cpu);
 122		int		(*multi)(unsigned int cpu,
 123					 struct hlist_node *node);
 124	} teardown;
 125	struct hlist_head	list;
 126	bool			cant_stop;
 127	bool			multi_instance;
 128};
 129
 130static DEFINE_MUTEX(cpuhp_state_mutex);
 131static struct cpuhp_step cpuhp_hp_states[];
 132
 133static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 134{
 135	return cpuhp_hp_states + state;
 136}
 137
 
 
 
 
 
 138/**
 139 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 140 * @cpu:	The cpu for which the callback should be invoked
 141 * @state:	The state to do callbacks for
 142 * @bringup:	True if the bringup callback should be invoked
 143 * @node:	For multi-instance, do a single entry callback for install/remove
 144 * @lastp:	For multi-instance rollback, remember how far we got
 145 *
 146 * Called from cpu hotplug and from the state register machinery.
 147 */
 148static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 149				 bool bringup, struct hlist_node *node,
 150				 struct hlist_node **lastp)
 151{
 152	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 153	struct cpuhp_step *step = cpuhp_get_step(state);
 154	int (*cbm)(unsigned int cpu, struct hlist_node *node);
 155	int (*cb)(unsigned int cpu);
 156	int ret, cnt;
 157
 158	if (st->fail == state) {
 159		st->fail = CPUHP_INVALID;
 160
 161		if (!(bringup ? step->startup.single : step->teardown.single))
 162			return 0;
 163
 164		return -EAGAIN;
 165	}
 166
 
 
 
 
 
 167	if (!step->multi_instance) {
 168		WARN_ON_ONCE(lastp && *lastp);
 169		cb = bringup ? step->startup.single : step->teardown.single;
 170		if (!cb)
 171			return 0;
 172		trace_cpuhp_enter(cpu, st->target, state, cb);
 173		ret = cb(cpu);
 174		trace_cpuhp_exit(cpu, st->state, state, ret);
 175		return ret;
 176	}
 177	cbm = bringup ? step->startup.multi : step->teardown.multi;
 178	if (!cbm)
 179		return 0;
 180
 181	/* Single invocation for instance add/remove */
 182	if (node) {
 183		WARN_ON_ONCE(lastp && *lastp);
 184		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 185		ret = cbm(cpu, node);
 186		trace_cpuhp_exit(cpu, st->state, state, ret);
 187		return ret;
 188	}
 189
 190	/* State transition. Invoke on all instances */
 191	cnt = 0;
 192	hlist_for_each(node, &step->list) {
 193		if (lastp && node == *lastp)
 194			break;
 195
 196		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 197		ret = cbm(cpu, node);
 198		trace_cpuhp_exit(cpu, st->state, state, ret);
 199		if (ret) {
 200			if (!lastp)
 201				goto err;
 202
 203			*lastp = node;
 204			return ret;
 205		}
 206		cnt++;
 207	}
 208	if (lastp)
 209		*lastp = NULL;
 210	return 0;
 211err:
 212	/* Rollback the instances if one failed */
 213	cbm = !bringup ? step->startup.multi : step->teardown.multi;
 214	if (!cbm)
 215		return ret;
 216
 217	hlist_for_each(node, &step->list) {
 218		if (!cnt--)
 219			break;
 220
 221		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 222		ret = cbm(cpu, node);
 223		trace_cpuhp_exit(cpu, st->state, state, ret);
 224		/*
 225		 * Rollback must not fail,
 226		 */
 227		WARN_ON_ONCE(ret);
 228	}
 229	return ret;
 230}
 231
 232#ifdef CONFIG_SMP
 233static bool cpuhp_is_ap_state(enum cpuhp_state state)
 234{
 235	/*
 236	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 237	 * purposes as that state is handled explicitly in cpu_down.
 238	 */
 239	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 240}
 241
 242static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 243{
 244	struct completion *done = bringup ? &st->done_up : &st->done_down;
 245	wait_for_completion(done);
 246}
 247
 248static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 249{
 250	struct completion *done = bringup ? &st->done_up : &st->done_down;
 251	complete(done);
 252}
 253
 254/*
 255 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 256 */
 257static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 258{
 259	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 260}
 261
 262/* Serializes the updates to cpu_online_mask, cpu_present_mask */
 263static DEFINE_MUTEX(cpu_add_remove_lock);
 264bool cpuhp_tasks_frozen;
 265EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 266
 267/*
 268 * The following two APIs (cpu_maps_update_begin/done) must be used when
 269 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 270 */
 271void cpu_maps_update_begin(void)
 272{
 273	mutex_lock(&cpu_add_remove_lock);
 274}
 275
 276void cpu_maps_update_done(void)
 277{
 278	mutex_unlock(&cpu_add_remove_lock);
 279}
 280
 281/*
 282 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 283 * Should always be manipulated under cpu_add_remove_lock
 284 */
 285static int cpu_hotplug_disabled;
 286
 287#ifdef CONFIG_HOTPLUG_CPU
 288
 289DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 290
 291void cpus_read_lock(void)
 292{
 293	percpu_down_read(&cpu_hotplug_lock);
 294}
 295EXPORT_SYMBOL_GPL(cpus_read_lock);
 296
 297int cpus_read_trylock(void)
 298{
 299	return percpu_down_read_trylock(&cpu_hotplug_lock);
 300}
 301EXPORT_SYMBOL_GPL(cpus_read_trylock);
 302
 303void cpus_read_unlock(void)
 304{
 305	percpu_up_read(&cpu_hotplug_lock);
 306}
 307EXPORT_SYMBOL_GPL(cpus_read_unlock);
 308
 309void cpus_write_lock(void)
 310{
 311	percpu_down_write(&cpu_hotplug_lock);
 312}
 313
 314void cpus_write_unlock(void)
 315{
 316	percpu_up_write(&cpu_hotplug_lock);
 317}
 318
 319void lockdep_assert_cpus_held(void)
 320{
 321	/*
 322	 * We can't have hotplug operations before userspace starts running,
 323	 * and some init codepaths will knowingly not take the hotplug lock.
 324	 * This is all valid, so mute lockdep until it makes sense to report
 325	 * unheld locks.
 326	 */
 327	if (system_state < SYSTEM_RUNNING)
 328		return;
 329
 330	percpu_rwsem_assert_held(&cpu_hotplug_lock);
 331}
 332
 
 
 
 
 
 
 
 333static void lockdep_acquire_cpus_lock(void)
 334{
 335	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
 336}
 337
 338static void lockdep_release_cpus_lock(void)
 339{
 340	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
 341}
 342
 343/*
 344 * Wait for currently running CPU hotplug operations to complete (if any) and
 345 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 346 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 347 * hotplug path before performing hotplug operations. So acquiring that lock
 348 * guarantees mutual exclusion from any currently running hotplug operations.
 349 */
 350void cpu_hotplug_disable(void)
 351{
 352	cpu_maps_update_begin();
 353	cpu_hotplug_disabled++;
 354	cpu_maps_update_done();
 355}
 356EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 357
 358static void __cpu_hotplug_enable(void)
 359{
 360	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 361		return;
 362	cpu_hotplug_disabled--;
 363}
 364
 365void cpu_hotplug_enable(void)
 366{
 367	cpu_maps_update_begin();
 368	__cpu_hotplug_enable();
 369	cpu_maps_update_done();
 370}
 371EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 372
 373#else
 374
 375static void lockdep_acquire_cpus_lock(void)
 376{
 377}
 378
 379static void lockdep_release_cpus_lock(void)
 380{
 381}
 382
 383#endif	/* CONFIG_HOTPLUG_CPU */
 384
 385/*
 386 * Architectures that need SMT-specific errata handling during SMT hotplug
 387 * should override this.
 388 */
 389void __weak arch_smt_update(void) { }
 390
 391#ifdef CONFIG_HOTPLUG_SMT
 392enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 393
 394void __init cpu_smt_disable(bool force)
 395{
 396	if (!cpu_smt_possible())
 397		return;
 398
 399	if (force) {
 400		pr_info("SMT: Force disabled\n");
 401		cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 402	} else {
 403		pr_info("SMT: disabled\n");
 404		cpu_smt_control = CPU_SMT_DISABLED;
 405	}
 406}
 407
 408/*
 409 * The decision whether SMT is supported can only be done after the full
 410 * CPU identification. Called from architecture code.
 411 */
 412void __init cpu_smt_check_topology(void)
 413{
 414	if (!topology_smt_supported())
 415		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 416}
 417
 418static int __init smt_cmdline_disable(char *str)
 419{
 420	cpu_smt_disable(str && !strcmp(str, "force"));
 421	return 0;
 422}
 423early_param("nosmt", smt_cmdline_disable);
 424
 425static inline bool cpu_smt_allowed(unsigned int cpu)
 426{
 427	if (cpu_smt_control == CPU_SMT_ENABLED)
 428		return true;
 429
 430	if (topology_is_primary_thread(cpu))
 431		return true;
 432
 433	/*
 434	 * On x86 it's required to boot all logical CPUs at least once so
 435	 * that the init code can get a chance to set CR4.MCE on each
 436	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
 437	 * core will shutdown the machine.
 438	 */
 439	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
 440}
 441
 442/* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
 443bool cpu_smt_possible(void)
 444{
 445	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
 446		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
 447}
 448EXPORT_SYMBOL_GPL(cpu_smt_possible);
 449#else
 450static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 451#endif
 452
 453static inline enum cpuhp_state
 454cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 455{
 456	enum cpuhp_state prev_state = st->state;
 
 457
 458	st->rollback = false;
 459	st->last = NULL;
 460
 461	st->target = target;
 462	st->single = false;
 463	st->bringup = st->state < target;
 
 
 464
 465	return prev_state;
 466}
 467
 468static inline void
 469cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 470{
 
 
 
 
 
 
 
 
 
 
 
 471	st->rollback = true;
 472
 473	/*
 474	 * If we have st->last we need to undo partial multi_instance of this
 475	 * state first. Otherwise start undo at the previous state.
 476	 */
 477	if (!st->last) {
 478		if (st->bringup)
 479			st->state--;
 480		else
 481			st->state++;
 482	}
 483
 484	st->target = prev_state;
 485	st->bringup = !st->bringup;
 
 486}
 487
 488/* Regular hotplug invocation of the AP hotplug thread */
 489static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 490{
 491	if (!st->single && st->state == st->target)
 492		return;
 493
 494	st->result = 0;
 495	/*
 496	 * Make sure the above stores are visible before should_run becomes
 497	 * true. Paired with the mb() above in cpuhp_thread_fun()
 498	 */
 499	smp_mb();
 500	st->should_run = true;
 501	wake_up_process(st->thread);
 502	wait_for_ap_thread(st, st->bringup);
 503}
 504
 505static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 506{
 507	enum cpuhp_state prev_state;
 508	int ret;
 509
 510	prev_state = cpuhp_set_state(st, target);
 511	__cpuhp_kick_ap(st);
 512	if ((ret = st->result)) {
 513		cpuhp_reset_state(st, prev_state);
 514		__cpuhp_kick_ap(st);
 515	}
 516
 517	return ret;
 518}
 519
 520static int bringup_wait_for_ap(unsigned int cpu)
 521{
 522	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 523
 524	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 525	wait_for_ap_thread(st, true);
 526	if (WARN_ON_ONCE((!cpu_online(cpu))))
 527		return -ECANCELED;
 528
 529	/* Unpark the hotplug thread of the target cpu */
 530	kthread_unpark(st->thread);
 531
 532	/*
 533	 * SMT soft disabling on X86 requires to bring the CPU out of the
 534	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 535	 * CPU marked itself as booted_once in notify_cpu_starting() so the
 536	 * cpu_smt_allowed() check will now return false if this is not the
 537	 * primary sibling.
 538	 */
 539	if (!cpu_smt_allowed(cpu))
 540		return -ECANCELED;
 541
 542	if (st->target <= CPUHP_AP_ONLINE_IDLE)
 543		return 0;
 544
 545	return cpuhp_kick_ap(st, st->target);
 546}
 547
 548static int bringup_cpu(unsigned int cpu)
 549{
 550	struct task_struct *idle = idle_thread_get(cpu);
 551	int ret;
 552
 553	/*
 554	 * Some architectures have to walk the irq descriptors to
 555	 * setup the vector space for the cpu which comes online.
 556	 * Prevent irq alloc/free across the bringup.
 557	 */
 558	irq_lock_sparse();
 559
 560	/* Arch-specific enabling code. */
 561	ret = __cpu_up(cpu, idle);
 562	irq_unlock_sparse();
 563	if (ret)
 564		return ret;
 565	return bringup_wait_for_ap(cpu);
 566}
 567
 568static int finish_cpu(unsigned int cpu)
 569{
 570	struct task_struct *idle = idle_thread_get(cpu);
 571	struct mm_struct *mm = idle->active_mm;
 572
 573	/*
 574	 * idle_task_exit() will have switched to &init_mm, now
 575	 * clean up any remaining active_mm state.
 576	 */
 577	if (mm != &init_mm)
 578		idle->active_mm = &init_mm;
 579	mmdrop(mm);
 580	return 0;
 581}
 582
 583/*
 584 * Hotplug state machine related functions
 585 */
 586
 587static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 
 
 
 
 
 
 
 
 
 
 588{
 589	for (st->state--; st->state > st->target; st->state--)
 590		cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 591}
 592
 593static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
 594{
 595	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
 596		return true;
 597	/*
 598	 * When CPU hotplug is disabled, then taking the CPU down is not
 599	 * possible because takedown_cpu() and the architecture and
 600	 * subsystem specific mechanisms are not available. So the CPU
 601	 * which would be completely unplugged again needs to stay around
 602	 * in the current state.
 603	 */
 604	return st->state <= CPUHP_BRINGUP_CPU;
 605}
 606
 607static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 608			      enum cpuhp_state target)
 609{
 610	enum cpuhp_state prev_state = st->state;
 611	int ret = 0;
 612
 613	while (st->state < target) {
 614		st->state++;
 615		ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 616		if (ret) {
 617			if (can_rollback_cpu(st)) {
 618				st->target = prev_state;
 619				undo_cpu_up(cpu, st);
 620			}
 621			break;
 622		}
 623	}
 624	return ret;
 625}
 626
 627/*
 628 * The cpu hotplug threads manage the bringup and teardown of the cpus
 629 */
 630static void cpuhp_create(unsigned int cpu)
 631{
 632	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 633
 634	init_completion(&st->done_up);
 635	init_completion(&st->done_down);
 
 636}
 637
 638static int cpuhp_should_run(unsigned int cpu)
 639{
 640	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 641
 642	return st->should_run;
 643}
 644
 645/*
 646 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 647 * callbacks when a state gets [un]installed at runtime.
 648 *
 649 * Each invocation of this function by the smpboot thread does a single AP
 650 * state callback.
 651 *
 652 * It has 3 modes of operation:
 653 *  - single: runs st->cb_state
 654 *  - up:     runs ++st->state, while st->state < st->target
 655 *  - down:   runs st->state--, while st->state > st->target
 656 *
 657 * When complete or on error, should_run is cleared and the completion is fired.
 658 */
 659static void cpuhp_thread_fun(unsigned int cpu)
 660{
 661	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 662	bool bringup = st->bringup;
 663	enum cpuhp_state state;
 664
 665	if (WARN_ON_ONCE(!st->should_run))
 666		return;
 667
 668	/*
 669	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 670	 * that if we see ->should_run we also see the rest of the state.
 671	 */
 672	smp_mb();
 673
 674	/*
 675	 * The BP holds the hotplug lock, but we're now running on the AP,
 676	 * ensure that anybody asserting the lock is held, will actually find
 677	 * it so.
 678	 */
 679	lockdep_acquire_cpus_lock();
 680	cpuhp_lock_acquire(bringup);
 681
 682	if (st->single) {
 683		state = st->cb_state;
 684		st->should_run = false;
 685	} else {
 686		if (bringup) {
 687			st->state++;
 688			state = st->state;
 689			st->should_run = (st->state < st->target);
 690			WARN_ON_ONCE(st->state > st->target);
 691		} else {
 692			state = st->state;
 693			st->state--;
 694			st->should_run = (st->state > st->target);
 695			WARN_ON_ONCE(st->state < st->target);
 696		}
 697	}
 698
 699	WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 700
 701	if (cpuhp_is_atomic_state(state)) {
 702		local_irq_disable();
 703		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 704		local_irq_enable();
 705
 706		/*
 707		 * STARTING/DYING must not fail!
 708		 */
 709		WARN_ON_ONCE(st->result);
 710	} else {
 711		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 712	}
 713
 714	if (st->result) {
 715		/*
 716		 * If we fail on a rollback, we're up a creek without no
 717		 * paddle, no way forward, no way back. We loose, thanks for
 718		 * playing.
 719		 */
 720		WARN_ON_ONCE(st->rollback);
 721		st->should_run = false;
 722	}
 723
 
 724	cpuhp_lock_release(bringup);
 725	lockdep_release_cpus_lock();
 726
 727	if (!st->should_run)
 728		complete_ap_thread(st, bringup);
 729}
 730
 731/* Invoke a single callback on a remote cpu */
 732static int
 733cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 734			 struct hlist_node *node)
 735{
 736	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 737	int ret;
 738
 739	if (!cpu_online(cpu))
 740		return 0;
 741
 742	cpuhp_lock_acquire(false);
 743	cpuhp_lock_release(false);
 744
 745	cpuhp_lock_acquire(true);
 746	cpuhp_lock_release(true);
 747
 748	/*
 749	 * If we are up and running, use the hotplug thread. For early calls
 750	 * we invoke the thread function directly.
 751	 */
 752	if (!st->thread)
 753		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 754
 755	st->rollback = false;
 756	st->last = NULL;
 757
 758	st->node = node;
 759	st->bringup = bringup;
 760	st->cb_state = state;
 761	st->single = true;
 762
 763	__cpuhp_kick_ap(st);
 764
 765	/*
 766	 * If we failed and did a partial, do a rollback.
 767	 */
 768	if ((ret = st->result) && st->last) {
 769		st->rollback = true;
 770		st->bringup = !bringup;
 771
 772		__cpuhp_kick_ap(st);
 773	}
 774
 775	/*
 776	 * Clean up the leftovers so the next hotplug operation wont use stale
 777	 * data.
 778	 */
 779	st->node = st->last = NULL;
 780	return ret;
 781}
 782
 783static int cpuhp_kick_ap_work(unsigned int cpu)
 784{
 785	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 786	enum cpuhp_state prev_state = st->state;
 787	int ret;
 788
 789	cpuhp_lock_acquire(false);
 790	cpuhp_lock_release(false);
 791
 792	cpuhp_lock_acquire(true);
 793	cpuhp_lock_release(true);
 794
 795	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 796	ret = cpuhp_kick_ap(st, st->target);
 797	trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 798
 799	return ret;
 800}
 801
 802static struct smp_hotplug_thread cpuhp_threads = {
 803	.store			= &cpuhp_state.thread,
 804	.create			= &cpuhp_create,
 805	.thread_should_run	= cpuhp_should_run,
 806	.thread_fn		= cpuhp_thread_fun,
 807	.thread_comm		= "cpuhp/%u",
 808	.selfparking		= true,
 809};
 810
 811void __init cpuhp_threads_init(void)
 812{
 813	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 814	kthread_unpark(this_cpu_read(cpuhp_state.thread));
 815}
 816
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 817#ifdef CONFIG_HOTPLUG_CPU
 
 
 
 
 818/**
 819 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 820 * @cpu: a CPU id
 821 *
 822 * This function walks all processes, finds a valid mm struct for each one and
 823 * then clears a corresponding bit in mm's cpumask.  While this all sounds
 824 * trivial, there are various non-obvious corner cases, which this function
 825 * tries to solve in a safe manner.
 826 *
 827 * Also note that the function uses a somewhat relaxed locking scheme, so it may
 828 * be called only for an already offlined CPU.
 829 */
 830void clear_tasks_mm_cpumask(int cpu)
 831{
 832	struct task_struct *p;
 833
 834	/*
 835	 * This function is called after the cpu is taken down and marked
 836	 * offline, so its not like new tasks will ever get this cpu set in
 837	 * their mm mask. -- Peter Zijlstra
 838	 * Thus, we may use rcu_read_lock() here, instead of grabbing
 839	 * full-fledged tasklist_lock.
 840	 */
 841	WARN_ON(cpu_online(cpu));
 842	rcu_read_lock();
 843	for_each_process(p) {
 844		struct task_struct *t;
 845
 846		/*
 847		 * Main thread might exit, but other threads may still have
 848		 * a valid mm. Find one.
 849		 */
 850		t = find_lock_task_mm(p);
 851		if (!t)
 852			continue;
 853		cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 854		task_unlock(t);
 855	}
 856	rcu_read_unlock();
 857}
 858
 859/* Take this CPU down. */
 860static int take_cpu_down(void *_param)
 861{
 862	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 863	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 864	int err, cpu = smp_processor_id();
 865	int ret;
 866
 867	/* Ensure this CPU doesn't handle any more interrupts. */
 868	err = __cpu_disable();
 869	if (err < 0)
 870		return err;
 871
 872	/*
 873	 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 874	 * do this step again.
 875	 */
 876	WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 877	st->state--;
 878	/* Invoke the former CPU_DYING callbacks */
 879	for (; st->state > target; st->state--) {
 880		ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 881		/*
 882		 * DYING must not fail!
 883		 */
 884		WARN_ON_ONCE(ret);
 885	}
 886
 887	/* Give up timekeeping duties */
 888	tick_handover_do_timer();
 889	/* Remove CPU from timer broadcasting */
 890	tick_offline_cpu(cpu);
 891	/* Park the stopper thread */
 892	stop_machine_park(cpu);
 893	return 0;
 894}
 895
 896static int takedown_cpu(unsigned int cpu)
 897{
 898	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 899	int err;
 900
 901	/* Park the smpboot threads */
 902	kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 903
 904	/*
 905	 * Prevent irq alloc/free while the dying cpu reorganizes the
 906	 * interrupt affinities.
 907	 */
 908	irq_lock_sparse();
 909
 910	/*
 911	 * So now all preempt/rcu users must observe !cpu_active().
 912	 */
 913	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 914	if (err) {
 915		/* CPU refused to die */
 916		irq_unlock_sparse();
 917		/* Unpark the hotplug thread so we can rollback there */
 918		kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 919		return err;
 920	}
 921	BUG_ON(cpu_online(cpu));
 922
 923	/*
 924	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
 925	 * all runnable tasks from the CPU, there's only the idle task left now
 926	 * that the migration thread is done doing the stop_machine thing.
 927	 *
 928	 * Wait for the stop thread to go away.
 929	 */
 930	wait_for_ap_thread(st, false);
 931	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 932
 933	/* Interrupts are moved away from the dying cpu, reenable alloc/free */
 934	irq_unlock_sparse();
 935
 936	hotplug_cpu__broadcast_tick_pull(cpu);
 937	/* This actually kills the CPU. */
 938	__cpu_die(cpu);
 939
 940	tick_cleanup_dead_cpu(cpu);
 941	rcutree_migrate_callbacks(cpu);
 942	return 0;
 943}
 944
 945static void cpuhp_complete_idle_dead(void *arg)
 946{
 947	struct cpuhp_cpu_state *st = arg;
 948
 949	complete_ap_thread(st, false);
 950}
 951
 952void cpuhp_report_idle_dead(void)
 953{
 954	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 955
 956	BUG_ON(st->state != CPUHP_AP_OFFLINE);
 957	rcu_report_dead(smp_processor_id());
 958	st->state = CPUHP_AP_IDLE_DEAD;
 959	/*
 960	 * We cannot call complete after rcu_report_dead() so we delegate it
 961	 * to an online cpu.
 962	 */
 963	smp_call_function_single(cpumask_first(cpu_online_mask),
 964				 cpuhp_complete_idle_dead, st, 0);
 965}
 966
 967static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 968{
 969	for (st->state++; st->state < st->target; st->state++)
 970		cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 971}
 972
 973static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 974				enum cpuhp_state target)
 975{
 976	enum cpuhp_state prev_state = st->state;
 977	int ret = 0;
 978
 979	for (; st->state > target; st->state--) {
 980		ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 981		if (ret) {
 982			st->target = prev_state;
 983			if (st->state < prev_state)
 984				undo_cpu_down(cpu, st);
 985			break;
 986		}
 987	}
 
 988	return ret;
 989}
 990
 991/* Requires cpu_add_remove_lock to be held */
 992static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 993			   enum cpuhp_state target)
 994{
 995	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 996	int prev_state, ret = 0;
 997
 998	if (num_online_cpus() == 1)
 999		return -EBUSY;
1000
1001	if (!cpu_present(cpu))
1002		return -EINVAL;
1003
1004	cpus_write_lock();
1005
1006	cpuhp_tasks_frozen = tasks_frozen;
1007
1008	prev_state = cpuhp_set_state(st, target);
1009	/*
1010	 * If the current CPU state is in the range of the AP hotplug thread,
1011	 * then we need to kick the thread.
1012	 */
1013	if (st->state > CPUHP_TEARDOWN_CPU) {
1014		st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1015		ret = cpuhp_kick_ap_work(cpu);
1016		/*
1017		 * The AP side has done the error rollback already. Just
1018		 * return the error code..
1019		 */
1020		if (ret)
1021			goto out;
1022
1023		/*
1024		 * We might have stopped still in the range of the AP hotplug
1025		 * thread. Nothing to do anymore.
1026		 */
1027		if (st->state > CPUHP_TEARDOWN_CPU)
1028			goto out;
1029
1030		st->target = target;
1031	}
1032	/*
1033	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1034	 * to do the further cleanups.
1035	 */
1036	ret = cpuhp_down_callbacks(cpu, st, target);
1037	if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1038		cpuhp_reset_state(st, prev_state);
1039		__cpuhp_kick_ap(st);
 
 
 
 
1040	}
1041
1042out:
1043	cpus_write_unlock();
1044	/*
1045	 * Do post unplug cleanup. This is still protected against
1046	 * concurrent CPU hotplug via cpu_add_remove_lock.
1047	 */
1048	lockup_detector_cleanup();
1049	arch_smt_update();
 
1050	return ret;
1051}
1052
1053static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1054{
1055	if (cpu_hotplug_disabled)
1056		return -EBUSY;
1057	return _cpu_down(cpu, 0, target);
1058}
1059
1060static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1061{
1062	int err;
1063
1064	cpu_maps_update_begin();
1065	err = cpu_down_maps_locked(cpu, target);
1066	cpu_maps_update_done();
1067	return err;
1068}
1069
1070/**
1071 * cpu_device_down - Bring down a cpu device
1072 * @dev: Pointer to the cpu device to offline
1073 *
1074 * This function is meant to be used by device core cpu subsystem only.
1075 *
1076 * Other subsystems should use remove_cpu() instead.
1077 */
1078int cpu_device_down(struct device *dev)
1079{
1080	return cpu_down(dev->id, CPUHP_OFFLINE);
1081}
1082
1083int remove_cpu(unsigned int cpu)
1084{
1085	int ret;
1086
1087	lock_device_hotplug();
1088	ret = device_offline(get_cpu_device(cpu));
1089	unlock_device_hotplug();
1090
1091	return ret;
1092}
1093EXPORT_SYMBOL_GPL(remove_cpu);
1094
1095void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1096{
1097	unsigned int cpu;
1098	int error;
1099
1100	cpu_maps_update_begin();
1101
1102	/*
1103	 * Make certain the cpu I'm about to reboot on is online.
1104	 *
1105	 * This is inline to what migrate_to_reboot_cpu() already do.
1106	 */
1107	if (!cpu_online(primary_cpu))
1108		primary_cpu = cpumask_first(cpu_online_mask);
1109
1110	for_each_online_cpu(cpu) {
1111		if (cpu == primary_cpu)
1112			continue;
1113
1114		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1115		if (error) {
1116			pr_err("Failed to offline CPU%d - error=%d",
1117				cpu, error);
1118			break;
1119		}
1120	}
1121
1122	/*
1123	 * Ensure all but the reboot CPU are offline.
1124	 */
1125	BUG_ON(num_online_cpus() > 1);
1126
1127	/*
1128	 * Make sure the CPUs won't be enabled by someone else after this
1129	 * point. Kexec will reboot to a new kernel shortly resetting
1130	 * everything along the way.
1131	 */
1132	cpu_hotplug_disabled++;
1133
1134	cpu_maps_update_done();
1135}
1136
1137#else
1138#define takedown_cpu		NULL
1139#endif /*CONFIG_HOTPLUG_CPU*/
1140
1141/**
1142 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1143 * @cpu: cpu that just started
1144 *
1145 * It must be called by the arch code on the new cpu, before the new cpu
1146 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1147 */
1148void notify_cpu_starting(unsigned int cpu)
1149{
1150	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1151	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1152	int ret;
1153
1154	rcu_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */
1155	cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1156	while (st->state < target) {
1157		st->state++;
1158		ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1159		/*
1160		 * STARTING must not fail!
1161		 */
1162		WARN_ON_ONCE(ret);
1163	}
1164}
1165
1166/*
1167 * Called from the idle task. Wake up the controlling task which brings the
1168 * hotplug thread of the upcoming CPU up and then delegates the rest of the
1169 * online bringup to the hotplug thread.
1170 */
1171void cpuhp_online_idle(enum cpuhp_state state)
1172{
1173	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1174
1175	/* Happens for the boot cpu */
1176	if (state != CPUHP_AP_ONLINE_IDLE)
1177		return;
1178
1179	/*
1180	 * Unpart the stopper thread before we start the idle loop (and start
1181	 * scheduling); this ensures the stopper task is always available.
1182	 */
1183	stop_machine_unpark(smp_processor_id());
1184
1185	st->state = CPUHP_AP_ONLINE_IDLE;
1186	complete_ap_thread(st, true);
1187}
1188
1189/* Requires cpu_add_remove_lock to be held */
1190static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1191{
1192	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1193	struct task_struct *idle;
1194	int ret = 0;
1195
1196	cpus_write_lock();
1197
1198	if (!cpu_present(cpu)) {
1199		ret = -EINVAL;
1200		goto out;
1201	}
1202
1203	/*
1204	 * The caller of cpu_up() might have raced with another
1205	 * caller. Nothing to do.
1206	 */
1207	if (st->state >= target)
1208		goto out;
1209
1210	if (st->state == CPUHP_OFFLINE) {
1211		/* Let it fail before we try to bring the cpu up */
1212		idle = idle_thread_get(cpu);
1213		if (IS_ERR(idle)) {
1214			ret = PTR_ERR(idle);
1215			goto out;
1216		}
1217	}
1218
1219	cpuhp_tasks_frozen = tasks_frozen;
1220
1221	cpuhp_set_state(st, target);
1222	/*
1223	 * If the current CPU state is in the range of the AP hotplug thread,
1224	 * then we need to kick the thread once more.
1225	 */
1226	if (st->state > CPUHP_BRINGUP_CPU) {
1227		ret = cpuhp_kick_ap_work(cpu);
1228		/*
1229		 * The AP side has done the error rollback already. Just
1230		 * return the error code..
1231		 */
1232		if (ret)
1233			goto out;
1234	}
1235
1236	/*
1237	 * Try to reach the target state. We max out on the BP at
1238	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1239	 * responsible for bringing it up to the target state.
1240	 */
1241	target = min((int)target, CPUHP_BRINGUP_CPU);
1242	ret = cpuhp_up_callbacks(cpu, st, target);
1243out:
1244	cpus_write_unlock();
1245	arch_smt_update();
 
1246	return ret;
1247}
1248
1249static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1250{
1251	int err = 0;
1252
1253	if (!cpu_possible(cpu)) {
1254		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1255		       cpu);
1256#if defined(CONFIG_IA64)
1257		pr_err("please check additional_cpus= boot parameter\n");
1258#endif
1259		return -EINVAL;
1260	}
1261
1262	err = try_online_node(cpu_to_node(cpu));
1263	if (err)
1264		return err;
1265
1266	cpu_maps_update_begin();
1267
1268	if (cpu_hotplug_disabled) {
1269		err = -EBUSY;
1270		goto out;
1271	}
1272	if (!cpu_smt_allowed(cpu)) {
1273		err = -EPERM;
1274		goto out;
1275	}
1276
1277	err = _cpu_up(cpu, 0, target);
1278out:
1279	cpu_maps_update_done();
1280	return err;
1281}
1282
1283/**
1284 * cpu_device_up - Bring up a cpu device
1285 * @dev: Pointer to the cpu device to online
1286 *
1287 * This function is meant to be used by device core cpu subsystem only.
1288 *
1289 * Other subsystems should use add_cpu() instead.
1290 */
1291int cpu_device_up(struct device *dev)
1292{
1293	return cpu_up(dev->id, CPUHP_ONLINE);
1294}
1295
1296int add_cpu(unsigned int cpu)
1297{
1298	int ret;
1299
1300	lock_device_hotplug();
1301	ret = device_online(get_cpu_device(cpu));
1302	unlock_device_hotplug();
1303
1304	return ret;
1305}
1306EXPORT_SYMBOL_GPL(add_cpu);
1307
1308/**
1309 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1310 * @sleep_cpu: The cpu we hibernated on and should be brought up.
1311 *
1312 * On some architectures like arm64, we can hibernate on any CPU, but on
1313 * wake up the CPU we hibernated on might be offline as a side effect of
1314 * using maxcpus= for example.
1315 */
1316int bringup_hibernate_cpu(unsigned int sleep_cpu)
1317{
1318	int ret;
1319
1320	if (!cpu_online(sleep_cpu)) {
1321		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1322		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1323		if (ret) {
1324			pr_err("Failed to bring hibernate-CPU up!\n");
1325			return ret;
1326		}
1327	}
1328	return 0;
1329}
1330
1331void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1332{
1333	unsigned int cpu;
1334
1335	for_each_present_cpu(cpu) {
1336		if (num_online_cpus() >= setup_max_cpus)
1337			break;
1338		if (!cpu_online(cpu))
1339			cpu_up(cpu, CPUHP_ONLINE);
1340	}
1341}
1342
1343#ifdef CONFIG_PM_SLEEP_SMP
1344static cpumask_var_t frozen_cpus;
1345
1346int freeze_secondary_cpus(int primary)
1347{
1348	int cpu, error = 0;
1349
1350	cpu_maps_update_begin();
1351	if (primary == -1) {
1352		primary = cpumask_first(cpu_online_mask);
1353		if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1354			primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1355	} else {
1356		if (!cpu_online(primary))
1357			primary = cpumask_first(cpu_online_mask);
1358	}
1359
1360	/*
1361	 * We take down all of the non-boot CPUs in one shot to avoid races
1362	 * with the userspace trying to use the CPU hotplug at the same time
1363	 */
1364	cpumask_clear(frozen_cpus);
1365
1366	pr_info("Disabling non-boot CPUs ...\n");
1367	for_each_online_cpu(cpu) {
1368		if (cpu == primary)
1369			continue;
1370
1371		if (pm_wakeup_pending()) {
1372			pr_info("Wakeup pending. Abort CPU freeze\n");
1373			error = -EBUSY;
1374			break;
1375		}
1376
1377		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1378		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1379		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1380		if (!error)
1381			cpumask_set_cpu(cpu, frozen_cpus);
1382		else {
1383			pr_err("Error taking CPU%d down: %d\n", cpu, error);
1384			break;
1385		}
1386	}
1387
1388	if (!error)
1389		BUG_ON(num_online_cpus() > 1);
1390	else
1391		pr_err("Non-boot CPUs are not disabled\n");
1392
1393	/*
1394	 * Make sure the CPUs won't be enabled by someone else. We need to do
1395	 * this even in case of failure as all freeze_secondary_cpus() users are
1396	 * supposed to do thaw_secondary_cpus() on the failure path.
1397	 */
1398	cpu_hotplug_disabled++;
1399
1400	cpu_maps_update_done();
1401	return error;
1402}
1403
1404void __weak arch_thaw_secondary_cpus_begin(void)
1405{
1406}
1407
1408void __weak arch_thaw_secondary_cpus_end(void)
1409{
1410}
1411
1412void thaw_secondary_cpus(void)
1413{
1414	int cpu, error;
1415
1416	/* Allow everyone to use the CPU hotplug again */
1417	cpu_maps_update_begin();
1418	__cpu_hotplug_enable();
1419	if (cpumask_empty(frozen_cpus))
1420		goto out;
1421
1422	pr_info("Enabling non-boot CPUs ...\n");
1423
1424	arch_thaw_secondary_cpus_begin();
1425
1426	for_each_cpu(cpu, frozen_cpus) {
1427		trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1428		error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1429		trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1430		if (!error) {
1431			pr_info("CPU%d is up\n", cpu);
1432			continue;
1433		}
1434		pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1435	}
1436
1437	arch_thaw_secondary_cpus_end();
1438
1439	cpumask_clear(frozen_cpus);
1440out:
1441	cpu_maps_update_done();
1442}
1443
1444static int __init alloc_frozen_cpus(void)
1445{
1446	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1447		return -ENOMEM;
1448	return 0;
1449}
1450core_initcall(alloc_frozen_cpus);
1451
1452/*
1453 * When callbacks for CPU hotplug notifications are being executed, we must
1454 * ensure that the state of the system with respect to the tasks being frozen
1455 * or not, as reported by the notification, remains unchanged *throughout the
1456 * duration* of the execution of the callbacks.
1457 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1458 *
1459 * This synchronization is implemented by mutually excluding regular CPU
1460 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1461 * Hibernate notifications.
1462 */
1463static int
1464cpu_hotplug_pm_callback(struct notifier_block *nb,
1465			unsigned long action, void *ptr)
1466{
1467	switch (action) {
1468
1469	case PM_SUSPEND_PREPARE:
1470	case PM_HIBERNATION_PREPARE:
1471		cpu_hotplug_disable();
1472		break;
1473
1474	case PM_POST_SUSPEND:
1475	case PM_POST_HIBERNATION:
1476		cpu_hotplug_enable();
1477		break;
1478
1479	default:
1480		return NOTIFY_DONE;
1481	}
1482
1483	return NOTIFY_OK;
1484}
1485
1486
1487static int __init cpu_hotplug_pm_sync_init(void)
1488{
1489	/*
1490	 * cpu_hotplug_pm_callback has higher priority than x86
1491	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1492	 * to disable cpu hotplug to avoid cpu hotplug race.
1493	 */
1494	pm_notifier(cpu_hotplug_pm_callback, 0);
1495	return 0;
1496}
1497core_initcall(cpu_hotplug_pm_sync_init);
1498
1499#endif /* CONFIG_PM_SLEEP_SMP */
1500
1501int __boot_cpu_id;
1502
1503#endif /* CONFIG_SMP */
1504
1505/* Boot processor state steps */
1506static struct cpuhp_step cpuhp_hp_states[] = {
1507	[CPUHP_OFFLINE] = {
1508		.name			= "offline",
1509		.startup.single		= NULL,
1510		.teardown.single	= NULL,
1511	},
1512#ifdef CONFIG_SMP
1513	[CPUHP_CREATE_THREADS]= {
1514		.name			= "threads:prepare",
1515		.startup.single		= smpboot_create_threads,
1516		.teardown.single	= NULL,
1517		.cant_stop		= true,
1518	},
1519	[CPUHP_PERF_PREPARE] = {
1520		.name			= "perf:prepare",
1521		.startup.single		= perf_event_init_cpu,
1522		.teardown.single	= perf_event_exit_cpu,
1523	},
1524	[CPUHP_WORKQUEUE_PREP] = {
1525		.name			= "workqueue:prepare",
1526		.startup.single		= workqueue_prepare_cpu,
1527		.teardown.single	= NULL,
1528	},
1529	[CPUHP_HRTIMERS_PREPARE] = {
1530		.name			= "hrtimers:prepare",
1531		.startup.single		= hrtimers_prepare_cpu,
1532		.teardown.single	= hrtimers_dead_cpu,
1533	},
1534	[CPUHP_SMPCFD_PREPARE] = {
1535		.name			= "smpcfd:prepare",
1536		.startup.single		= smpcfd_prepare_cpu,
1537		.teardown.single	= smpcfd_dead_cpu,
1538	},
1539	[CPUHP_RELAY_PREPARE] = {
1540		.name			= "relay:prepare",
1541		.startup.single		= relay_prepare_cpu,
1542		.teardown.single	= NULL,
1543	},
1544	[CPUHP_SLAB_PREPARE] = {
1545		.name			= "slab:prepare",
1546		.startup.single		= slab_prepare_cpu,
1547		.teardown.single	= slab_dead_cpu,
1548	},
1549	[CPUHP_RCUTREE_PREP] = {
1550		.name			= "RCU/tree:prepare",
1551		.startup.single		= rcutree_prepare_cpu,
1552		.teardown.single	= rcutree_dead_cpu,
1553	},
1554	/*
1555	 * On the tear-down path, timers_dead_cpu() must be invoked
1556	 * before blk_mq_queue_reinit_notify() from notify_dead(),
1557	 * otherwise a RCU stall occurs.
1558	 */
1559	[CPUHP_TIMERS_PREPARE] = {
1560		.name			= "timers:prepare",
1561		.startup.single		= timers_prepare_cpu,
1562		.teardown.single	= timers_dead_cpu,
1563	},
1564	/* Kicks the plugged cpu into life */
1565	[CPUHP_BRINGUP_CPU] = {
1566		.name			= "cpu:bringup",
1567		.startup.single		= bringup_cpu,
1568		.teardown.single	= finish_cpu,
1569		.cant_stop		= true,
1570	},
1571	/* Final state before CPU kills itself */
1572	[CPUHP_AP_IDLE_DEAD] = {
1573		.name			= "idle:dead",
1574	},
1575	/*
1576	 * Last state before CPU enters the idle loop to die. Transient state
1577	 * for synchronization.
1578	 */
1579	[CPUHP_AP_OFFLINE] = {
1580		.name			= "ap:offline",
1581		.cant_stop		= true,
1582	},
1583	/* First state is scheduler control. Interrupts are disabled */
1584	[CPUHP_AP_SCHED_STARTING] = {
1585		.name			= "sched:starting",
1586		.startup.single		= sched_cpu_starting,
1587		.teardown.single	= sched_cpu_dying,
1588	},
1589	[CPUHP_AP_RCUTREE_DYING] = {
1590		.name			= "RCU/tree:dying",
1591		.startup.single		= NULL,
1592		.teardown.single	= rcutree_dying_cpu,
1593	},
1594	[CPUHP_AP_SMPCFD_DYING] = {
1595		.name			= "smpcfd:dying",
1596		.startup.single		= NULL,
1597		.teardown.single	= smpcfd_dying_cpu,
1598	},
1599	/* Entry state on starting. Interrupts enabled from here on. Transient
1600	 * state for synchronsization */
1601	[CPUHP_AP_ONLINE] = {
1602		.name			= "ap:online",
1603	},
1604	/*
1605	 * Handled on controll processor until the plugged processor manages
1606	 * this itself.
1607	 */
1608	[CPUHP_TEARDOWN_CPU] = {
1609		.name			= "cpu:teardown",
1610		.startup.single		= NULL,
1611		.teardown.single	= takedown_cpu,
1612		.cant_stop		= true,
1613	},
 
 
 
 
 
 
 
1614	/* Handle smpboot threads park/unpark */
1615	[CPUHP_AP_SMPBOOT_THREADS] = {
1616		.name			= "smpboot/threads:online",
1617		.startup.single		= smpboot_unpark_threads,
1618		.teardown.single	= smpboot_park_threads,
1619	},
1620	[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1621		.name			= "irq/affinity:online",
1622		.startup.single		= irq_affinity_online_cpu,
1623		.teardown.single	= NULL,
1624	},
1625	[CPUHP_AP_PERF_ONLINE] = {
1626		.name			= "perf:online",
1627		.startup.single		= perf_event_init_cpu,
1628		.teardown.single	= perf_event_exit_cpu,
1629	},
1630	[CPUHP_AP_WATCHDOG_ONLINE] = {
1631		.name			= "lockup_detector:online",
1632		.startup.single		= lockup_detector_online_cpu,
1633		.teardown.single	= lockup_detector_offline_cpu,
1634	},
1635	[CPUHP_AP_WORKQUEUE_ONLINE] = {
1636		.name			= "workqueue:online",
1637		.startup.single		= workqueue_online_cpu,
1638		.teardown.single	= workqueue_offline_cpu,
1639	},
1640	[CPUHP_AP_RCUTREE_ONLINE] = {
1641		.name			= "RCU/tree:online",
1642		.startup.single		= rcutree_online_cpu,
1643		.teardown.single	= rcutree_offline_cpu,
1644	},
1645#endif
1646	/*
1647	 * The dynamically registered state space is here
1648	 */
1649
1650#ifdef CONFIG_SMP
1651	/* Last state is scheduler control setting the cpu active */
1652	[CPUHP_AP_ACTIVE] = {
1653		.name			= "sched:active",
1654		.startup.single		= sched_cpu_activate,
1655		.teardown.single	= sched_cpu_deactivate,
1656	},
1657#endif
1658
1659	/* CPU is fully up and running. */
1660	[CPUHP_ONLINE] = {
1661		.name			= "online",
1662		.startup.single		= NULL,
1663		.teardown.single	= NULL,
1664	},
1665};
1666
1667/* Sanity check for callbacks */
1668static int cpuhp_cb_check(enum cpuhp_state state)
1669{
1670	if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1671		return -EINVAL;
1672	return 0;
1673}
1674
1675/*
1676 * Returns a free for dynamic slot assignment of the Online state. The states
1677 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1678 * by having no name assigned.
1679 */
1680static int cpuhp_reserve_state(enum cpuhp_state state)
1681{
1682	enum cpuhp_state i, end;
1683	struct cpuhp_step *step;
1684
1685	switch (state) {
1686	case CPUHP_AP_ONLINE_DYN:
1687		step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1688		end = CPUHP_AP_ONLINE_DYN_END;
1689		break;
1690	case CPUHP_BP_PREPARE_DYN:
1691		step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1692		end = CPUHP_BP_PREPARE_DYN_END;
1693		break;
1694	default:
1695		return -EINVAL;
1696	}
1697
1698	for (i = state; i <= end; i++, step++) {
1699		if (!step->name)
1700			return i;
1701	}
1702	WARN(1, "No more dynamic states available for CPU hotplug\n");
1703	return -ENOSPC;
1704}
1705
1706static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1707				 int (*startup)(unsigned int cpu),
1708				 int (*teardown)(unsigned int cpu),
1709				 bool multi_instance)
1710{
1711	/* (Un)Install the callbacks for further cpu hotplug operations */
1712	struct cpuhp_step *sp;
1713	int ret = 0;
1714
1715	/*
1716	 * If name is NULL, then the state gets removed.
1717	 *
1718	 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1719	 * the first allocation from these dynamic ranges, so the removal
1720	 * would trigger a new allocation and clear the wrong (already
1721	 * empty) state, leaving the callbacks of the to be cleared state
1722	 * dangling, which causes wreckage on the next hotplug operation.
1723	 */
1724	if (name && (state == CPUHP_AP_ONLINE_DYN ||
1725		     state == CPUHP_BP_PREPARE_DYN)) {
1726		ret = cpuhp_reserve_state(state);
1727		if (ret < 0)
1728			return ret;
1729		state = ret;
1730	}
1731	sp = cpuhp_get_step(state);
1732	if (name && sp->name)
1733		return -EBUSY;
1734
1735	sp->startup.single = startup;
1736	sp->teardown.single = teardown;
1737	sp->name = name;
1738	sp->multi_instance = multi_instance;
1739	INIT_HLIST_HEAD(&sp->list);
1740	return ret;
1741}
1742
1743static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1744{
1745	return cpuhp_get_step(state)->teardown.single;
1746}
1747
1748/*
1749 * Call the startup/teardown function for a step either on the AP or
1750 * on the current CPU.
1751 */
1752static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1753			    struct hlist_node *node)
1754{
1755	struct cpuhp_step *sp = cpuhp_get_step(state);
1756	int ret;
1757
1758	/*
1759	 * If there's nothing to do, we done.
1760	 * Relies on the union for multi_instance.
1761	 */
1762	if ((bringup && !sp->startup.single) ||
1763	    (!bringup && !sp->teardown.single))
1764		return 0;
1765	/*
1766	 * The non AP bound callbacks can fail on bringup. On teardown
1767	 * e.g. module removal we crash for now.
1768	 */
1769#ifdef CONFIG_SMP
1770	if (cpuhp_is_ap_state(state))
1771		ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1772	else
1773		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1774#else
1775	ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1776#endif
1777	BUG_ON(ret && !bringup);
1778	return ret;
1779}
1780
1781/*
1782 * Called from __cpuhp_setup_state on a recoverable failure.
1783 *
1784 * Note: The teardown callbacks for rollback are not allowed to fail!
1785 */
1786static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1787				   struct hlist_node *node)
1788{
1789	int cpu;
1790
1791	/* Roll back the already executed steps on the other cpus */
1792	for_each_present_cpu(cpu) {
1793		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1794		int cpustate = st->state;
1795
1796		if (cpu >= failedcpu)
1797			break;
1798
1799		/* Did we invoke the startup call on that cpu ? */
1800		if (cpustate >= state)
1801			cpuhp_issue_call(cpu, state, false, node);
1802	}
1803}
1804
1805int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1806					  struct hlist_node *node,
1807					  bool invoke)
1808{
1809	struct cpuhp_step *sp;
1810	int cpu;
1811	int ret;
1812
1813	lockdep_assert_cpus_held();
1814
1815	sp = cpuhp_get_step(state);
1816	if (sp->multi_instance == false)
1817		return -EINVAL;
1818
1819	mutex_lock(&cpuhp_state_mutex);
1820
1821	if (!invoke || !sp->startup.multi)
1822		goto add_node;
1823
1824	/*
1825	 * Try to call the startup callback for each present cpu
1826	 * depending on the hotplug state of the cpu.
1827	 */
1828	for_each_present_cpu(cpu) {
1829		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1830		int cpustate = st->state;
1831
1832		if (cpustate < state)
1833			continue;
1834
1835		ret = cpuhp_issue_call(cpu, state, true, node);
1836		if (ret) {
1837			if (sp->teardown.multi)
1838				cpuhp_rollback_install(cpu, state, node);
1839			goto unlock;
1840		}
1841	}
1842add_node:
1843	ret = 0;
1844	hlist_add_head(node, &sp->list);
1845unlock:
1846	mutex_unlock(&cpuhp_state_mutex);
1847	return ret;
1848}
1849
1850int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1851			       bool invoke)
1852{
1853	int ret;
1854
1855	cpus_read_lock();
1856	ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1857	cpus_read_unlock();
1858	return ret;
1859}
1860EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1861
1862/**
1863 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1864 * @state:		The state to setup
1865 * @invoke:		If true, the startup function is invoked for cpus where
1866 *			cpu state >= @state
1867 * @startup:		startup callback function
1868 * @teardown:		teardown callback function
1869 * @multi_instance:	State is set up for multiple instances which get
1870 *			added afterwards.
1871 *
1872 * The caller needs to hold cpus read locked while calling this function.
1873 * Returns:
1874 *   On success:
1875 *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1876 *      0 for all other states
1877 *   On failure: proper (negative) error code
1878 */
1879int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1880				   const char *name, bool invoke,
1881				   int (*startup)(unsigned int cpu),
1882				   int (*teardown)(unsigned int cpu),
1883				   bool multi_instance)
1884{
1885	int cpu, ret = 0;
1886	bool dynstate;
1887
1888	lockdep_assert_cpus_held();
1889
1890	if (cpuhp_cb_check(state) || !name)
1891		return -EINVAL;
1892
1893	mutex_lock(&cpuhp_state_mutex);
1894
1895	ret = cpuhp_store_callbacks(state, name, startup, teardown,
1896				    multi_instance);
1897
1898	dynstate = state == CPUHP_AP_ONLINE_DYN;
1899	if (ret > 0 && dynstate) {
1900		state = ret;
1901		ret = 0;
1902	}
1903
1904	if (ret || !invoke || !startup)
1905		goto out;
1906
1907	/*
1908	 * Try to call the startup callback for each present cpu
1909	 * depending on the hotplug state of the cpu.
1910	 */
1911	for_each_present_cpu(cpu) {
1912		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1913		int cpustate = st->state;
1914
1915		if (cpustate < state)
1916			continue;
1917
1918		ret = cpuhp_issue_call(cpu, state, true, NULL);
1919		if (ret) {
1920			if (teardown)
1921				cpuhp_rollback_install(cpu, state, NULL);
1922			cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1923			goto out;
1924		}
1925	}
1926out:
1927	mutex_unlock(&cpuhp_state_mutex);
1928	/*
1929	 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1930	 * dynamically allocated state in case of success.
1931	 */
1932	if (!ret && dynstate)
1933		return state;
1934	return ret;
1935}
1936EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1937
1938int __cpuhp_setup_state(enum cpuhp_state state,
1939			const char *name, bool invoke,
1940			int (*startup)(unsigned int cpu),
1941			int (*teardown)(unsigned int cpu),
1942			bool multi_instance)
1943{
1944	int ret;
1945
1946	cpus_read_lock();
1947	ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1948					     teardown, multi_instance);
1949	cpus_read_unlock();
1950	return ret;
1951}
1952EXPORT_SYMBOL(__cpuhp_setup_state);
1953
1954int __cpuhp_state_remove_instance(enum cpuhp_state state,
1955				  struct hlist_node *node, bool invoke)
1956{
1957	struct cpuhp_step *sp = cpuhp_get_step(state);
1958	int cpu;
1959
1960	BUG_ON(cpuhp_cb_check(state));
1961
1962	if (!sp->multi_instance)
1963		return -EINVAL;
1964
1965	cpus_read_lock();
1966	mutex_lock(&cpuhp_state_mutex);
1967
1968	if (!invoke || !cpuhp_get_teardown_cb(state))
1969		goto remove;
1970	/*
1971	 * Call the teardown callback for each present cpu depending
1972	 * on the hotplug state of the cpu. This function is not
1973	 * allowed to fail currently!
1974	 */
1975	for_each_present_cpu(cpu) {
1976		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1977		int cpustate = st->state;
1978
1979		if (cpustate >= state)
1980			cpuhp_issue_call(cpu, state, false, node);
1981	}
1982
1983remove:
1984	hlist_del(node);
1985	mutex_unlock(&cpuhp_state_mutex);
1986	cpus_read_unlock();
1987
1988	return 0;
1989}
1990EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1991
1992/**
1993 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1994 * @state:	The state to remove
1995 * @invoke:	If true, the teardown function is invoked for cpus where
1996 *		cpu state >= @state
1997 *
1998 * The caller needs to hold cpus read locked while calling this function.
1999 * The teardown callback is currently not allowed to fail. Think
2000 * about module removal!
2001 */
2002void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2003{
2004	struct cpuhp_step *sp = cpuhp_get_step(state);
2005	int cpu;
2006
2007	BUG_ON(cpuhp_cb_check(state));
2008
2009	lockdep_assert_cpus_held();
2010
2011	mutex_lock(&cpuhp_state_mutex);
2012	if (sp->multi_instance) {
2013		WARN(!hlist_empty(&sp->list),
2014		     "Error: Removing state %d which has instances left.\n",
2015		     state);
2016		goto remove;
2017	}
2018
2019	if (!invoke || !cpuhp_get_teardown_cb(state))
2020		goto remove;
2021
2022	/*
2023	 * Call the teardown callback for each present cpu depending
2024	 * on the hotplug state of the cpu. This function is not
2025	 * allowed to fail currently!
2026	 */
2027	for_each_present_cpu(cpu) {
2028		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2029		int cpustate = st->state;
2030
2031		if (cpustate >= state)
2032			cpuhp_issue_call(cpu, state, false, NULL);
2033	}
2034remove:
2035	cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2036	mutex_unlock(&cpuhp_state_mutex);
2037}
2038EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2039
2040void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2041{
2042	cpus_read_lock();
2043	__cpuhp_remove_state_cpuslocked(state, invoke);
2044	cpus_read_unlock();
2045}
2046EXPORT_SYMBOL(__cpuhp_remove_state);
2047
2048#ifdef CONFIG_HOTPLUG_SMT
2049static void cpuhp_offline_cpu_device(unsigned int cpu)
2050{
2051	struct device *dev = get_cpu_device(cpu);
2052
2053	dev->offline = true;
2054	/* Tell user space about the state change */
2055	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2056}
2057
2058static void cpuhp_online_cpu_device(unsigned int cpu)
2059{
2060	struct device *dev = get_cpu_device(cpu);
2061
2062	dev->offline = false;
2063	/* Tell user space about the state change */
2064	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2065}
2066
2067int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2068{
2069	int cpu, ret = 0;
2070
2071	cpu_maps_update_begin();
2072	for_each_online_cpu(cpu) {
2073		if (topology_is_primary_thread(cpu))
2074			continue;
2075		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2076		if (ret)
2077			break;
2078		/*
2079		 * As this needs to hold the cpu maps lock it's impossible
2080		 * to call device_offline() because that ends up calling
2081		 * cpu_down() which takes cpu maps lock. cpu maps lock
2082		 * needs to be held as this might race against in kernel
2083		 * abusers of the hotplug machinery (thermal management).
2084		 *
2085		 * So nothing would update device:offline state. That would
2086		 * leave the sysfs entry stale and prevent onlining after
2087		 * smt control has been changed to 'off' again. This is
2088		 * called under the sysfs hotplug lock, so it is properly
2089		 * serialized against the regular offline usage.
2090		 */
2091		cpuhp_offline_cpu_device(cpu);
2092	}
2093	if (!ret)
2094		cpu_smt_control = ctrlval;
2095	cpu_maps_update_done();
2096	return ret;
2097}
2098
2099int cpuhp_smt_enable(void)
2100{
2101	int cpu, ret = 0;
2102
2103	cpu_maps_update_begin();
2104	cpu_smt_control = CPU_SMT_ENABLED;
2105	for_each_present_cpu(cpu) {
2106		/* Skip online CPUs and CPUs on offline nodes */
2107		if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2108			continue;
2109		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2110		if (ret)
2111			break;
2112		/* See comment in cpuhp_smt_disable() */
2113		cpuhp_online_cpu_device(cpu);
2114	}
2115	cpu_maps_update_done();
2116	return ret;
2117}
2118#endif
2119
2120#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2121static ssize_t show_cpuhp_state(struct device *dev,
2122				struct device_attribute *attr, char *buf)
2123{
2124	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2125
2126	return sprintf(buf, "%d\n", st->state);
2127}
2128static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
2129
2130static ssize_t write_cpuhp_target(struct device *dev,
2131				  struct device_attribute *attr,
2132				  const char *buf, size_t count)
2133{
2134	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2135	struct cpuhp_step *sp;
2136	int target, ret;
2137
2138	ret = kstrtoint(buf, 10, &target);
2139	if (ret)
2140		return ret;
2141
2142#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2143	if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2144		return -EINVAL;
2145#else
2146	if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2147		return -EINVAL;
2148#endif
2149
2150	ret = lock_device_hotplug_sysfs();
2151	if (ret)
2152		return ret;
2153
2154	mutex_lock(&cpuhp_state_mutex);
2155	sp = cpuhp_get_step(target);
2156	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2157	mutex_unlock(&cpuhp_state_mutex);
2158	if (ret)
2159		goto out;
2160
2161	if (st->state < target)
2162		ret = cpu_up(dev->id, target);
2163	else
2164		ret = cpu_down(dev->id, target);
2165out:
2166	unlock_device_hotplug();
2167	return ret ? ret : count;
2168}
2169
2170static ssize_t show_cpuhp_target(struct device *dev,
2171				 struct device_attribute *attr, char *buf)
2172{
2173	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2174
2175	return sprintf(buf, "%d\n", st->target);
2176}
2177static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
2178
2179
2180static ssize_t write_cpuhp_fail(struct device *dev,
2181				struct device_attribute *attr,
2182				const char *buf, size_t count)
2183{
2184	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2185	struct cpuhp_step *sp;
2186	int fail, ret;
2187
2188	ret = kstrtoint(buf, 10, &fail);
2189	if (ret)
2190		return ret;
2191
 
 
 
 
 
2192	if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2193		return -EINVAL;
2194
2195	/*
2196	 * Cannot fail STARTING/DYING callbacks.
2197	 */
2198	if (cpuhp_is_atomic_state(fail))
2199		return -EINVAL;
2200
2201	/*
 
 
 
 
 
 
 
 
 
2202	 * Cannot fail anything that doesn't have callbacks.
2203	 */
2204	mutex_lock(&cpuhp_state_mutex);
2205	sp = cpuhp_get_step(fail);
2206	if (!sp->startup.single && !sp->teardown.single)
2207		ret = -EINVAL;
2208	mutex_unlock(&cpuhp_state_mutex);
2209	if (ret)
2210		return ret;
2211
2212	st->fail = fail;
2213
2214	return count;
2215}
2216
2217static ssize_t show_cpuhp_fail(struct device *dev,
2218			       struct device_attribute *attr, char *buf)
2219{
2220	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2221
2222	return sprintf(buf, "%d\n", st->fail);
2223}
2224
2225static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2226
2227static struct attribute *cpuhp_cpu_attrs[] = {
2228	&dev_attr_state.attr,
2229	&dev_attr_target.attr,
2230	&dev_attr_fail.attr,
2231	NULL
2232};
2233
2234static const struct attribute_group cpuhp_cpu_attr_group = {
2235	.attrs = cpuhp_cpu_attrs,
2236	.name = "hotplug",
2237	NULL
2238};
2239
2240static ssize_t show_cpuhp_states(struct device *dev,
2241				 struct device_attribute *attr, char *buf)
2242{
2243	ssize_t cur, res = 0;
2244	int i;
2245
2246	mutex_lock(&cpuhp_state_mutex);
2247	for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2248		struct cpuhp_step *sp = cpuhp_get_step(i);
2249
2250		if (sp->name) {
2251			cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2252			buf += cur;
2253			res += cur;
2254		}
2255	}
2256	mutex_unlock(&cpuhp_state_mutex);
2257	return res;
2258}
2259static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2260
2261static struct attribute *cpuhp_cpu_root_attrs[] = {
2262	&dev_attr_states.attr,
2263	NULL
2264};
2265
2266static const struct attribute_group cpuhp_cpu_root_attr_group = {
2267	.attrs = cpuhp_cpu_root_attrs,
2268	.name = "hotplug",
2269	NULL
2270};
2271
2272#ifdef CONFIG_HOTPLUG_SMT
2273
2274static ssize_t
2275__store_smt_control(struct device *dev, struct device_attribute *attr,
2276		    const char *buf, size_t count)
2277{
2278	int ctrlval, ret;
2279
2280	if (sysfs_streq(buf, "on"))
2281		ctrlval = CPU_SMT_ENABLED;
2282	else if (sysfs_streq(buf, "off"))
2283		ctrlval = CPU_SMT_DISABLED;
2284	else if (sysfs_streq(buf, "forceoff"))
2285		ctrlval = CPU_SMT_FORCE_DISABLED;
2286	else
2287		return -EINVAL;
2288
2289	if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2290		return -EPERM;
2291
2292	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2293		return -ENODEV;
2294
2295	ret = lock_device_hotplug_sysfs();
2296	if (ret)
2297		return ret;
2298
2299	if (ctrlval != cpu_smt_control) {
2300		switch (ctrlval) {
2301		case CPU_SMT_ENABLED:
2302			ret = cpuhp_smt_enable();
2303			break;
2304		case CPU_SMT_DISABLED:
2305		case CPU_SMT_FORCE_DISABLED:
2306			ret = cpuhp_smt_disable(ctrlval);
2307			break;
2308		}
2309	}
2310
2311	unlock_device_hotplug();
2312	return ret ? ret : count;
2313}
2314
2315#else /* !CONFIG_HOTPLUG_SMT */
2316static ssize_t
2317__store_smt_control(struct device *dev, struct device_attribute *attr,
2318		    const char *buf, size_t count)
2319{
2320	return -ENODEV;
2321}
2322#endif /* CONFIG_HOTPLUG_SMT */
2323
2324static const char *smt_states[] = {
2325	[CPU_SMT_ENABLED]		= "on",
2326	[CPU_SMT_DISABLED]		= "off",
2327	[CPU_SMT_FORCE_DISABLED]	= "forceoff",
2328	[CPU_SMT_NOT_SUPPORTED]		= "notsupported",
2329	[CPU_SMT_NOT_IMPLEMENTED]	= "notimplemented",
2330};
2331
2332static ssize_t
2333show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2334{
2335	const char *state = smt_states[cpu_smt_control];
2336
2337	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2338}
2339
2340static ssize_t
2341store_smt_control(struct device *dev, struct device_attribute *attr,
2342		  const char *buf, size_t count)
2343{
2344	return __store_smt_control(dev, attr, buf, count);
2345}
2346static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2347
2348static ssize_t
2349show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2350{
2351	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2352}
2353static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2354
2355static struct attribute *cpuhp_smt_attrs[] = {
2356	&dev_attr_control.attr,
2357	&dev_attr_active.attr,
2358	NULL
2359};
2360
2361static const struct attribute_group cpuhp_smt_attr_group = {
2362	.attrs = cpuhp_smt_attrs,
2363	.name = "smt",
2364	NULL
2365};
2366
2367static int __init cpu_smt_sysfs_init(void)
2368{
2369	return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2370				  &cpuhp_smt_attr_group);
2371}
2372
2373static int __init cpuhp_sysfs_init(void)
2374{
2375	int cpu, ret;
2376
2377	ret = cpu_smt_sysfs_init();
2378	if (ret)
2379		return ret;
2380
2381	ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2382				 &cpuhp_cpu_root_attr_group);
2383	if (ret)
2384		return ret;
2385
2386	for_each_possible_cpu(cpu) {
2387		struct device *dev = get_cpu_device(cpu);
2388
2389		if (!dev)
2390			continue;
2391		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2392		if (ret)
2393			return ret;
2394	}
2395	return 0;
2396}
2397device_initcall(cpuhp_sysfs_init);
2398#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2399
2400/*
2401 * cpu_bit_bitmap[] is a special, "compressed" data structure that
2402 * represents all NR_CPUS bits binary values of 1<<nr.
2403 *
2404 * It is used by cpumask_of() to get a constant address to a CPU
2405 * mask value that has a single bit set only.
2406 */
2407
2408/* cpu_bit_bitmap[0] is empty - so we can back into it */
2409#define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
2410#define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2411#define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2412#define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2413
2414const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2415
2416	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
2417	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
2418#if BITS_PER_LONG > 32
2419	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
2420	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
2421#endif
2422};
2423EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2424
2425const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2426EXPORT_SYMBOL(cpu_all_bits);
2427
2428#ifdef CONFIG_INIT_ALL_POSSIBLE
2429struct cpumask __cpu_possible_mask __read_mostly
2430	= {CPU_BITS_ALL};
2431#else
2432struct cpumask __cpu_possible_mask __read_mostly;
2433#endif
2434EXPORT_SYMBOL(__cpu_possible_mask);
2435
2436struct cpumask __cpu_online_mask __read_mostly;
2437EXPORT_SYMBOL(__cpu_online_mask);
2438
2439struct cpumask __cpu_present_mask __read_mostly;
2440EXPORT_SYMBOL(__cpu_present_mask);
2441
2442struct cpumask __cpu_active_mask __read_mostly;
2443EXPORT_SYMBOL(__cpu_active_mask);
 
 
 
2444
2445atomic_t __num_online_cpus __read_mostly;
2446EXPORT_SYMBOL(__num_online_cpus);
2447
2448void init_cpu_present(const struct cpumask *src)
2449{
2450	cpumask_copy(&__cpu_present_mask, src);
2451}
2452
2453void init_cpu_possible(const struct cpumask *src)
2454{
2455	cpumask_copy(&__cpu_possible_mask, src);
2456}
2457
2458void init_cpu_online(const struct cpumask *src)
2459{
2460	cpumask_copy(&__cpu_online_mask, src);
2461}
2462
2463void set_cpu_online(unsigned int cpu, bool online)
2464{
2465	/*
2466	 * atomic_inc/dec() is required to handle the horrid abuse of this
2467	 * function by the reboot and kexec code which invoke it from
2468	 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2469	 * regular CPU hotplug is properly serialized.
2470	 *
2471	 * Note, that the fact that __num_online_cpus is of type atomic_t
2472	 * does not protect readers which are not serialized against
2473	 * concurrent hotplug operations.
2474	 */
2475	if (online) {
2476		if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2477			atomic_inc(&__num_online_cpus);
2478	} else {
2479		if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2480			atomic_dec(&__num_online_cpus);
2481	}
2482}
2483
2484/*
2485 * Activate the first processor.
2486 */
2487void __init boot_cpu_init(void)
2488{
2489	int cpu = smp_processor_id();
2490
2491	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
2492	set_cpu_online(cpu, true);
2493	set_cpu_active(cpu, true);
2494	set_cpu_present(cpu, true);
2495	set_cpu_possible(cpu, true);
2496
2497#ifdef CONFIG_SMP
2498	__boot_cpu_id = cpu;
2499#endif
2500}
2501
2502/*
2503 * Must be called _AFTER_ setting up the per_cpu areas
2504 */
2505void __init boot_cpu_hotplug_init(void)
2506{
2507#ifdef CONFIG_SMP
2508	cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2509#endif
2510	this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2511}
2512
2513/*
2514 * These are used for a global "mitigations=" cmdline option for toggling
2515 * optional CPU mitigations.
2516 */
2517enum cpu_mitigations {
2518	CPU_MITIGATIONS_OFF,
2519	CPU_MITIGATIONS_AUTO,
2520	CPU_MITIGATIONS_AUTO_NOSMT,
2521};
2522
2523static enum cpu_mitigations cpu_mitigations __ro_after_init =
2524	CPU_MITIGATIONS_AUTO;
2525
2526static int __init mitigations_parse_cmdline(char *arg)
2527{
2528	if (!strcmp(arg, "off"))
2529		cpu_mitigations = CPU_MITIGATIONS_OFF;
2530	else if (!strcmp(arg, "auto"))
2531		cpu_mitigations = CPU_MITIGATIONS_AUTO;
2532	else if (!strcmp(arg, "auto,nosmt"))
2533		cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2534	else
2535		pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2536			arg);
2537
2538	return 0;
2539}
2540early_param("mitigations", mitigations_parse_cmdline);
2541
2542/* mitigations=off */
2543bool cpu_mitigations_off(void)
2544{
2545	return cpu_mitigations == CPU_MITIGATIONS_OFF;
2546}
2547EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2548
2549/* mitigations=auto,nosmt */
2550bool cpu_mitigations_auto_nosmt(void)
2551{
2552	return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2553}
2554EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);