1// SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3 *	x86 SMP booting functions
   4 *
   5 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
   6 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
   7 *	Copyright 2001 Andi Kleen, SuSE Labs.
   8 *
   9 *	Much of the core SMP work is based on previous work by Thomas Radke, to
  10 *	whom a great many thanks are extended.
  11 *
  12 *	Thanks to Intel for making available several different Pentium,
  13 *	Pentium Pro and Pentium-II/Xeon MP machines.
  14 *	Original development of Linux SMP code supported by Caldera.
  15 *
 
 
 
  16 *	Fixes
  17 *		Felix Koop	:	NR_CPUS used properly
  18 *		Jose Renau	:	Handle single CPU case.
  19 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
  20 *		Greg Wright	:	Fix for kernel stacks panic.
  21 *		Erich Boleyn	:	MP v1.4 and additional changes.
  22 *	Matthias Sattler	:	Changes for 2.1 kernel map.
  23 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
  24 *	Michael Chastain	:	Change trampoline.S to gnu as.
  25 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
  26 *		Ingo Molnar	:	Added APIC timers, based on code
  27 *					from Jose Renau
  28 *		Ingo Molnar	:	various cleanups and rewrites
  29 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
  30 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
  31 *	Andi Kleen		:	Changed for SMP boot into long mode.
  32 *		Martin J. Bligh	: 	Added support for multi-quad systems
  33 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
  34 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
  35 *      Andi Kleen              :       Converted to new state machine.
  36 *	Ashok Raj		: 	CPU hotplug support
  37 *	Glauber Costa		:	i386 and x86_64 integration
  38 */
  39
  40#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  41
  42#include <linux/init.h>
  43#include <linux/smp.h>
  44#include <linux/export.h>
  45#include <linux/sched.h>
  46#include <linux/sched/topology.h>
  47#include <linux/sched/hotplug.h>
  48#include <linux/sched/task_stack.h>
  49#include <linux/percpu.h>
  50#include <linux/memblock.h>
  51#include <linux/err.h>
  52#include <linux/nmi.h>
  53#include <linux/tboot.h>
  54#include <linux/stackprotector.h>
  55#include <linux/gfp.h>
  56#include <linux/cpuidle.h>
  57#include <linux/numa.h>
  58
  59#include <asm/acpi.h>
  60#include <asm/desc.h>
  61#include <asm/nmi.h>
  62#include <asm/irq.h>
 
  63#include <asm/realmode.h>
  64#include <asm/cpu.h>
  65#include <asm/numa.h>
  66#include <asm/pgtable.h>
  67#include <asm/tlbflush.h>
  68#include <asm/mtrr.h>
  69#include <asm/mwait.h>
  70#include <asm/apic.h>
  71#include <asm/io_apic.h>
  72#include <asm/fpu/internal.h>
 
  73#include <asm/setup.h>
  74#include <asm/uv/uv.h>
  75#include <linux/mc146818rtc.h>
 
  76#include <asm/i8259.h>
 
  77#include <asm/misc.h>
  78#include <asm/qspinlock.h>
  79#include <asm/intel-family.h>
  80#include <asm/cpu_device_id.h>
  81#include <asm/spec-ctrl.h>
  82#include <asm/hw_irq.h>
 
 
 
 
 
  83
  84/* representing HT siblings of each logical CPU */
  85DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
  86EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  87
  88/* representing HT and core siblings of each logical CPU */
  89DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
  90EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  91
  92/* representing HT, core, and die siblings of each logical CPU */
  93DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map);
  94EXPORT_PER_CPU_SYMBOL(cpu_die_map);
  95
  96DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
  97
  98/* Per CPU bogomips and other parameters */
  99DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
 100EXPORT_PER_CPU_SYMBOL(cpu_info);
 101
 102/* Logical package management. We might want to allocate that dynamically */
 103unsigned int __max_logical_packages __read_mostly;
 104EXPORT_SYMBOL(__max_logical_packages);
 105static unsigned int logical_packages __read_mostly;
 106static unsigned int logical_die __read_mostly;
 107
 108/* Maximum number of SMT threads on any online core */
 109int __read_mostly __max_smt_threads = 1;
 110
 111/* Flag to indicate if a complete sched domain rebuild is required */
 112bool x86_topology_update;
 113
 114int arch_update_cpu_topology(void)
 115{
 116	int retval = x86_topology_update;
 117
 118	x86_topology_update = false;
 119	return retval;
 120}
 121
 122static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
 123{
 124	unsigned long flags;
 125
 126	spin_lock_irqsave(&rtc_lock, flags);
 127	CMOS_WRITE(0xa, 0xf);
 128	spin_unlock_irqrestore(&rtc_lock, flags);
 129	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
 130							start_eip >> 4;
 131	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
 132							start_eip & 0xf;
 133}
 134
 135static inline void smpboot_restore_warm_reset_vector(void)
 136{
 137	unsigned long flags;
 138
 139	/*
 140	 * Paranoid:  Set warm reset code and vector here back
 141	 * to default values.
 142	 */
 143	spin_lock_irqsave(&rtc_lock, flags);
 144	CMOS_WRITE(0, 0xf);
 145	spin_unlock_irqrestore(&rtc_lock, flags);
 146
 147	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
 148}
 149
 150/*
 151 * Report back to the Boot Processor during boot time or to the caller processor
 152 * during CPU online.
 153 */
 154static void smp_callin(void)
 155{
 156	int cpuid;
 
 157
 158	/*
 159	 * If waken up by an INIT in an 82489DX configuration
 160	 * cpu_callout_mask guarantees we don't get here before
 161	 * an INIT_deassert IPI reaches our local APIC, so it is
 162	 * now safe to touch our local APIC.
 
 
 163	 */
 164	cpuid = smp_processor_id();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 165
 166	/*
 167	 * the boot CPU has finished the init stage and is spinning
 168	 * on callin_map until we finish. We are free to set up this
 169	 * CPU, first the APIC. (this is probably redundant on most
 170	 * boards)
 171	 */
 172	apic_ap_setup();
 
 
 
 
 
 173
 174	/*
 175	 * Save our processor parameters. Note: this information
 176	 * is needed for clock calibration.
 177	 */
 178	smp_store_cpu_info(cpuid);
 179
 180	/*
 181	 * The topology information must be up to date before
 182	 * calibrate_delay() and notify_cpu_starting().
 183	 */
 184	set_cpu_sibling_map(raw_smp_processor_id());
 185
 186	/*
 187	 * Get our bogomips.
 188	 * Update loops_per_jiffy in cpu_data. Previous call to
 189	 * smp_store_cpu_info() stored a value that is close but not as
 190	 * accurate as the value just calculated.
 191	 */
 192	calibrate_delay();
 193	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
 194	pr_debug("Stack at about %p\n", &cpuid);
 195
 
 
 
 
 
 196	wmb();
 197
 198	notify_cpu_starting(cpuid);
 199
 200	/*
 201	 * Allow the master to continue.
 202	 */
 203	cpumask_set_cpu(cpuid, cpu_callin_mask);
 204}
 205
 206static int cpu0_logical_apicid;
 207static int enable_start_cpu0;
 208/*
 209 * Activate a secondary processor.
 210 */
 211static void notrace start_secondary(void *unused)
 212{
 213	/*
 214	 * Don't put *anything* except direct CPU state initialization
 215	 * before cpu_init(), SMP booting is too fragile that we want to
 216	 * limit the things done here to the most necessary things.
 217	 */
 218	cr4_init();
 219
 220#ifdef CONFIG_X86_32
 221	/* switch away from the initial page table */
 222	load_cr3(swapper_pg_dir);
 223	__flush_tlb_all();
 224#endif
 225	load_current_idt();
 226	cpu_init();
 227	x86_cpuinit.early_percpu_clock_init();
 228	preempt_disable();
 229	smp_callin();
 230
 231	enable_start_cpu0 = 0;
 232
 
 
 
 
 
 
 233	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 234	barrier();
 235	/*
 236	 * Check TSC synchronization with the boot CPU:
 237	 */
 238	check_tsc_sync_target();
 239
 240	speculative_store_bypass_ht_init();
 241
 242	/*
 243	 * Lock vector_lock, set CPU online and bring the vector
 244	 * allocator online. Online must be set with vector_lock held
 245	 * to prevent a concurrent irq setup/teardown from seeing a
 246	 * half valid vector space.
 247	 */
 248	lock_vector_lock();
 249	set_cpu_online(smp_processor_id(), true);
 250	lapic_online();
 251	unlock_vector_lock();
 252	cpu_set_state_online(smp_processor_id());
 253	x86_platform.nmi_init();
 254
 255	/* enable local interrupts */
 256	local_irq_enable();
 257
 258	/* to prevent fake stack check failure in clock setup */
 259	boot_init_stack_canary();
 260
 261	x86_cpuinit.setup_percpu_clockev();
 262
 263	wmb();
 264	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 265}
 266
 267/**
 268 * topology_is_primary_thread - Check whether CPU is the primary SMT thread
 269 * @cpu:	CPU to check
 270 */
 271bool topology_is_primary_thread(unsigned int cpu)
 272{
 273	return apic_id_is_primary_thread(per_cpu(x86_cpu_to_apicid, cpu));
 274}
 275
 276/**
 277 * topology_smt_supported - Check whether SMT is supported by the CPUs
 278 */
 279bool topology_smt_supported(void)
 280{
 281	return smp_num_siblings > 1;
 282}
 283
 284/**
 285 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 286 *
 287 * Returns logical package id or -1 if not found
 288 */
 289int topology_phys_to_logical_pkg(unsigned int phys_pkg)
 290{
 291	int cpu;
 292
 293	for_each_possible_cpu(cpu) {
 294		struct cpuinfo_x86 *c = &cpu_data(cpu);
 295
 296		if (c->initialized && c->phys_proc_id == phys_pkg)
 297			return c->logical_proc_id;
 298	}
 299	return -1;
 300}
 301EXPORT_SYMBOL(topology_phys_to_logical_pkg);
 302/**
 303 * topology_phys_to_logical_die - Map a physical die id to logical
 304 *
 305 * Returns logical die id or -1 if not found
 306 */
 307int topology_phys_to_logical_die(unsigned int die_id, unsigned int cur_cpu)
 308{
 309	int cpu;
 310	int proc_id = cpu_data(cur_cpu).phys_proc_id;
 311
 312	for_each_possible_cpu(cpu) {
 313		struct cpuinfo_x86 *c = &cpu_data(cpu);
 314
 315		if (c->initialized && c->cpu_die_id == die_id &&
 316		    c->phys_proc_id == proc_id)
 317			return c->logical_die_id;
 318	}
 319	return -1;
 320}
 321EXPORT_SYMBOL(topology_phys_to_logical_die);
 322
 323/**
 324 * topology_update_package_map - Update the physical to logical package map
 325 * @pkg:	The physical package id as retrieved via CPUID
 326 * @cpu:	The cpu for which this is updated
 327 */
 328int topology_update_package_map(unsigned int pkg, unsigned int cpu)
 329{
 330	int new;
 331
 332	/* Already available somewhere? */
 333	new = topology_phys_to_logical_pkg(pkg);
 334	if (new >= 0)
 335		goto found;
 336
 337	new = logical_packages++;
 338	if (new != pkg) {
 339		pr_info("CPU %u Converting physical %u to logical package %u\n",
 340			cpu, pkg, new);
 341	}
 342found:
 343	cpu_data(cpu).logical_proc_id = new;
 344	return 0;
 345}
 346/**
 347 * topology_update_die_map - Update the physical to logical die map
 348 * @die:	The die id as retrieved via CPUID
 349 * @cpu:	The cpu for which this is updated
 350 */
 351int topology_update_die_map(unsigned int die, unsigned int cpu)
 352{
 353	int new;
 354
 355	/* Already available somewhere? */
 356	new = topology_phys_to_logical_die(die, cpu);
 357	if (new >= 0)
 358		goto found;
 359
 360	new = logical_die++;
 361	if (new != die) {
 362		pr_info("CPU %u Converting physical %u to logical die %u\n",
 363			cpu, die, new);
 364	}
 365found:
 366	cpu_data(cpu).logical_die_id = new;
 367	return 0;
 368}
 369
 370void __init smp_store_boot_cpu_info(void)
 371{
 372	int id = 0; /* CPU 0 */
 373	struct cpuinfo_x86 *c = &cpu_data(id);
 374
 375	*c = boot_cpu_data;
 376	c->cpu_index = id;
 377	topology_update_package_map(c->phys_proc_id, id);
 378	topology_update_die_map(c->cpu_die_id, id);
 379	c->initialized = true;
 380}
 381
 382/*
 383 * The bootstrap kernel entry code has set these up. Save them for
 384 * a given CPU
 385 */
 386void smp_store_cpu_info(int id)
 387{
 388	struct cpuinfo_x86 *c = &cpu_data(id);
 389
 390	/* Copy boot_cpu_data only on the first bringup */
 391	if (!c->initialized)
 392		*c = boot_cpu_data;
 393	c->cpu_index = id;
 394	/*
 395	 * During boot time, CPU0 has this setup already. Save the info when
 396	 * bringing up AP or offlined CPU0.
 397	 */
 398	identify_secondary_cpu(c);
 399	c->initialized = true;
 400}
 401
 402static bool
 403topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 404{
 405	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 406
 407	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
 408}
 409
 410static bool
 411topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
 412{
 413	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 414
 415	return !WARN_ONCE(!topology_same_node(c, o),
 416		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
 417		"[node: %d != %d]. Ignoring dependency.\n",
 418		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
 419}
 420
 421#define link_mask(mfunc, c1, c2)					\
 422do {									\
 423	cpumask_set_cpu((c1), mfunc(c2));				\
 424	cpumask_set_cpu((c2), mfunc(c1));				\
 425} while (0)
 426
 427static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 428{
 429	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
 430		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 431
 432		if (c->phys_proc_id == o->phys_proc_id &&
 433		    c->cpu_die_id == o->cpu_die_id &&
 434		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2)) {
 435			if (c->cpu_core_id == o->cpu_core_id)
 436				return topology_sane(c, o, "smt");
 437
 438			if ((c->cu_id != 0xff) &&
 439			    (o->cu_id != 0xff) &&
 440			    (c->cu_id == o->cu_id))
 441				return topology_sane(c, o, "smt");
 442		}
 443
 444	} else if (c->phys_proc_id == o->phys_proc_id &&
 445		   c->cpu_die_id == o->cpu_die_id &&
 446		   c->cpu_core_id == o->cpu_core_id) {
 447		return topology_sane(c, o, "smt");
 448	}
 449
 450	return false;
 451}
 452
 453/*
 454 * Define snc_cpu[] for SNC (Sub-NUMA Cluster) CPUs.
 455 *
 456 * These are Intel CPUs that enumerate an LLC that is shared by
 457 * multiple NUMA nodes. The LLC on these systems is shared for
 458 * off-package data access but private to the NUMA node (half
 459 * of the package) for on-package access.
 460 *
 461 * CPUID (the source of the information about the LLC) can only
 462 * enumerate the cache as being shared *or* unshared, but not
 463 * this particular configuration. The CPU in this case enumerates
 464 * the cache to be shared across the entire package (spanning both
 465 * NUMA nodes).
 466 */
 467
 468static const struct x86_cpu_id snc_cpu[] = {
 469	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_SKYLAKE_X },
 470	{}
 471};
 472
 473static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 474{
 475	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 476
 477	/* Do not match if we do not have a valid APICID for cpu: */
 478	if (per_cpu(cpu_llc_id, cpu1) == BAD_APICID)
 479		return false;
 480
 481	/* Do not match if LLC id does not match: */
 482	if (per_cpu(cpu_llc_id, cpu1) != per_cpu(cpu_llc_id, cpu2))
 483		return false;
 484
 485	/*
 486	 * Allow the SNC topology without warning. Return of false
 487	 * means 'c' does not share the LLC of 'o'. This will be
 488	 * reflected to userspace.
 489	 */
 490	if (!topology_same_node(c, o) && x86_match_cpu(snc_cpu))
 491		return false;
 492
 493	return topology_sane(c, o, "llc");
 494}
 495
 496/*
 497 * Unlike the other levels, we do not enforce keeping a
 498 * multicore group inside a NUMA node.  If this happens, we will
 499 * discard the MC level of the topology later.
 500 */
 501static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 502{
 503	if (c->phys_proc_id == o->phys_proc_id)
 504		return true;
 505	return false;
 506}
 507
 508static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 509{
 510	if ((c->phys_proc_id == o->phys_proc_id) &&
 511		(c->cpu_die_id == o->cpu_die_id))
 512		return true;
 513	return false;
 514}
 515
 516
 517#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
 518static inline int x86_sched_itmt_flags(void)
 519{
 520	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;
 521}
 
 522
 523#ifdef CONFIG_SCHED_MC
 524static int x86_core_flags(void)
 525{
 526	return cpu_core_flags() | x86_sched_itmt_flags();
 527}
 528#endif
 529#ifdef CONFIG_SCHED_SMT
 530static int x86_smt_flags(void)
 531{
 532	return cpu_smt_flags() | x86_sched_itmt_flags();
 533}
 534#endif
 535#endif
 536
 537static struct sched_domain_topology_level x86_numa_in_package_topology[] = {
 538#ifdef CONFIG_SCHED_SMT
 539	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 540#endif
 541#ifdef CONFIG_SCHED_MC
 542	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 543#endif
 544	{ NULL, },
 545};
 546
 547static struct sched_domain_topology_level x86_topology[] = {
 548#ifdef CONFIG_SCHED_SMT
 549	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 550#endif
 551#ifdef CONFIG_SCHED_MC
 552	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 553#endif
 554	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
 555	{ NULL, },
 556};
 557
 558/*
 559 * Set if a package/die has multiple NUMA nodes inside.
 560 * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
 561 * Sub-NUMA Clustering have this.
 562 */
 563static bool x86_has_numa_in_package;
 564
 565void set_cpu_sibling_map(int cpu)
 566{
 567	bool has_smt = smp_num_siblings > 1;
 568	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
 569	struct cpuinfo_x86 *c = &cpu_data(cpu);
 570	struct cpuinfo_x86 *o;
 571	int i, threads;
 572
 573	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
 574
 575	if (!has_mp) {
 576		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
 577		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
 578		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
 579		cpumask_set_cpu(cpu, topology_die_cpumask(cpu));
 580		c->booted_cores = 1;
 581		return;
 582	}
 583
 584	for_each_cpu(i, cpu_sibling_setup_mask) {
 585		o = &cpu_data(i);
 586
 587		if ((i == cpu) || (has_smt && match_smt(c, o)))
 588			link_mask(topology_sibling_cpumask, cpu, i);
 589
 590		if ((i == cpu) || (has_mp && match_llc(c, o)))
 591			link_mask(cpu_llc_shared_mask, cpu, i);
 592
 593	}
 594
 595	/*
 596	 * This needs a separate iteration over the cpus because we rely on all
 597	 * topology_sibling_cpumask links to be set-up.
 598	 */
 599	for_each_cpu(i, cpu_sibling_setup_mask) {
 600		o = &cpu_data(i);
 601
 602		if ((i == cpu) || (has_mp && match_pkg(c, o))) {
 603			link_mask(topology_core_cpumask, cpu, i);
 604
 605			/*
 606			 *  Does this new cpu bringup a new core?
 607			 */
 608			if (cpumask_weight(
 609			    topology_sibling_cpumask(cpu)) == 1) {
 610				/*
 611				 * for each core in package, increment
 612				 * the booted_cores for this new cpu
 613				 */
 614				if (cpumask_first(
 615				    topology_sibling_cpumask(i)) == i)
 616					c->booted_cores++;
 617				/*
 618				 * increment the core count for all
 619				 * the other cpus in this package
 620				 */
 621				if (i != cpu)
 622					cpu_data(i).booted_cores++;
 623			} else if (i != cpu && !c->booted_cores)
 624				c->booted_cores = cpu_data(i).booted_cores;
 625		}
 626		if (match_pkg(c, o) && !topology_same_node(c, o))
 627			x86_has_numa_in_package = true;
 628
 629		if ((i == cpu) || (has_mp && match_die(c, o)))
 630			link_mask(topology_die_cpumask, cpu, i);
 631	}
 632
 633	threads = cpumask_weight(topology_sibling_cpumask(cpu));
 634	if (threads > __max_smt_threads)
 635		__max_smt_threads = threads;
 636}
 637
 638/* maps the cpu to the sched domain representing multi-core */
 639const struct cpumask *cpu_coregroup_mask(int cpu)
 640{
 641	return cpu_llc_shared_mask(cpu);
 642}
 643
 644static void impress_friends(void)
 645{
 646	int cpu;
 647	unsigned long bogosum = 0;
 648	/*
 649	 * Allow the user to impress friends.
 650	 */
 651	pr_debug("Before bogomips\n");
 652	for_each_possible_cpu(cpu)
 653		if (cpumask_test_cpu(cpu, cpu_callout_mask))
 654			bogosum += cpu_data(cpu).loops_per_jiffy;
 655	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
 656		num_online_cpus(),
 657		bogosum/(500000/HZ),
 658		(bogosum/(5000/HZ))%100);
 659
 660	pr_debug("Before bogocount - setting activated=1\n");
 661}
 662
 663void __inquire_remote_apic(int apicid)
 664{
 665	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
 666	const char * const names[] = { "ID", "VERSION", "SPIV" };
 667	int timeout;
 668	u32 status;
 669
 670	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
 671
 672	for (i = 0; i < ARRAY_SIZE(regs); i++) {
 673		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
 674
 675		/*
 676		 * Wait for idle.
 677		 */
 678		status = safe_apic_wait_icr_idle();
 679		if (status)
 680			pr_cont("a previous APIC delivery may have failed\n");
 681
 682		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
 683
 684		timeout = 0;
 685		do {
 686			udelay(100);
 687			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
 688		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
 689
 690		switch (status) {
 691		case APIC_ICR_RR_VALID:
 692			status = apic_read(APIC_RRR);
 693			pr_cont("%08x\n", status);
 694			break;
 695		default:
 696			pr_cont("failed\n");
 697		}
 698	}
 699}
 700
 701/*
 702 * The Multiprocessor Specification 1.4 (1997) example code suggests
 703 * that there should be a 10ms delay between the BSP asserting INIT
 704 * and de-asserting INIT, when starting a remote processor.
 705 * But that slows boot and resume on modern processors, which include
 706 * many cores and don't require that delay.
 707 *
 708 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
 709 * Modern processor families are quirked to remove the delay entirely.
 710 */
 711#define UDELAY_10MS_DEFAULT 10000
 712
 713static unsigned int init_udelay = UINT_MAX;
 714
 715static int __init cpu_init_udelay(char *str)
 716{
 717	get_option(&str, &init_udelay);
 718
 719	return 0;
 720}
 721early_param("cpu_init_udelay", cpu_init_udelay);
 722
 723static void __init smp_quirk_init_udelay(void)
 724{
 725	/* if cmdline changed it from default, leave it alone */
 726	if (init_udelay != UINT_MAX)
 727		return;
 728
 729	/* if modern processor, use no delay */
 730	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
 731	    ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && (boot_cpu_data.x86 >= 0x18)) ||
 732	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
 733		init_udelay = 0;
 734		return;
 735	}
 736	/* else, use legacy delay */
 737	init_udelay = UDELAY_10MS_DEFAULT;
 738}
 739
 740/*
 741 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 742 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 743 * won't ... remember to clear down the APIC, etc later.
 744 */
 745int
 746wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
 747{
 748	unsigned long send_status, accept_status = 0;
 749	int maxlvt;
 750
 751	/* Target chip */
 752	/* Boot on the stack */
 753	/* Kick the second */
 754	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
 755
 756	pr_debug("Waiting for send to finish...\n");
 757	send_status = safe_apic_wait_icr_idle();
 758
 759	/*
 760	 * Give the other CPU some time to accept the IPI.
 761	 */
 762	udelay(200);
 763	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 764		maxlvt = lapic_get_maxlvt();
 765		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
 766			apic_write(APIC_ESR, 0);
 767		accept_status = (apic_read(APIC_ESR) & 0xEF);
 768	}
 769	pr_debug("NMI sent\n");
 770
 771	if (send_status)
 772		pr_err("APIC never delivered???\n");
 773	if (accept_status)
 774		pr_err("APIC delivery error (%lx)\n", accept_status);
 775
 776	return (send_status | accept_status);
 777}
 778
 779static int
 780wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
 781{
 782	unsigned long send_status = 0, accept_status = 0;
 783	int maxlvt, num_starts, j;
 784
 785	maxlvt = lapic_get_maxlvt();
 786
 787	/*
 788	 * Be paranoid about clearing APIC errors.
 789	 */
 790	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 791		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 792			apic_write(APIC_ESR, 0);
 793		apic_read(APIC_ESR);
 794	}
 795
 796	pr_debug("Asserting INIT\n");
 797
 798	/*
 799	 * Turn INIT on target chip
 800	 */
 801	/*
 802	 * Send IPI
 803	 */
 804	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
 805		       phys_apicid);
 806
 807	pr_debug("Waiting for send to finish...\n");
 808	send_status = safe_apic_wait_icr_idle();
 809
 810	udelay(init_udelay);
 811
 812	pr_debug("Deasserting INIT\n");
 813
 814	/* Target chip */
 815	/* Send IPI */
 816	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
 817
 818	pr_debug("Waiting for send to finish...\n");
 819	send_status = safe_apic_wait_icr_idle();
 820
 821	mb();
 
 822
 823	/*
 824	 * Should we send STARTUP IPIs ?
 825	 *
 826	 * Determine this based on the APIC version.
 827	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
 828	 */
 829	if (APIC_INTEGRATED(boot_cpu_apic_version))
 830		num_starts = 2;
 831	else
 832		num_starts = 0;
 833
 834	/*
 
 
 
 
 
 
 
 835	 * Run STARTUP IPI loop.
 836	 */
 837	pr_debug("#startup loops: %d\n", num_starts);
 838
 839	for (j = 1; j <= num_starts; j++) {
 840		pr_debug("Sending STARTUP #%d\n", j);
 841		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 842			apic_write(APIC_ESR, 0);
 843		apic_read(APIC_ESR);
 844		pr_debug("After apic_write\n");
 845
 846		/*
 847		 * STARTUP IPI
 848		 */
 849
 850		/* Target chip */
 851		/* Boot on the stack */
 852		/* Kick the second */
 853		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
 854			       phys_apicid);
 855
 856		/*
 857		 * Give the other CPU some time to accept the IPI.
 858		 */
 859		if (init_udelay == 0)
 860			udelay(10);
 861		else
 862			udelay(300);
 863
 864		pr_debug("Startup point 1\n");
 865
 866		pr_debug("Waiting for send to finish...\n");
 867		send_status = safe_apic_wait_icr_idle();
 868
 869		/*
 870		 * Give the other CPU some time to accept the IPI.
 871		 */
 872		if (init_udelay == 0)
 873			udelay(10);
 874		else
 875			udelay(200);
 876
 877		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 878			apic_write(APIC_ESR, 0);
 879		accept_status = (apic_read(APIC_ESR) & 0xEF);
 880		if (send_status || accept_status)
 881			break;
 882	}
 883	pr_debug("After Startup\n");
 884
 885	if (send_status)
 886		pr_err("APIC never delivered???\n");
 887	if (accept_status)
 888		pr_err("APIC delivery error (%lx)\n", accept_status);
 889
 890	return (send_status | accept_status);
 891}
 892
 
 
 
 
 
 
 
 
 893/* reduce the number of lines printed when booting a large cpu count system */
 894static void announce_cpu(int cpu, int apicid)
 895{
 896	static int current_node = NUMA_NO_NODE;
 897	int node = early_cpu_to_node(cpu);
 898	static int width, node_width;
 899
 900	if (!width)
 901		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
 902
 903	if (!node_width)
 904		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */
 905
 906	if (cpu == 1)
 907		printk(KERN_INFO "x86: Booting SMP configuration:\n");
 908
 909	if (system_state < SYSTEM_RUNNING) {
 910		if (node != current_node) {
 911			if (current_node > (-1))
 912				pr_cont("\n");
 913			current_node = node;
 914
 915			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
 916			       node_width - num_digits(node), " ", node);
 917		}
 918
 919		/* Add padding for the BSP */
 920		if (cpu == 1)
 921			pr_cont("%*s", width + 1, " ");
 922
 923		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);
 924
 925	} else
 926		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
 927			node, cpu, apicid);
 928}
 929
 930static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
 931{
 932	int cpu;
 933
 934	cpu = smp_processor_id();
 935	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
 936		return NMI_HANDLED;
 937
 938	return NMI_DONE;
 939}
 940
 941/*
 942 * Wake up AP by INIT, INIT, STARTUP sequence.
 943 *
 944 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 945 * boot-strap code which is not a desired behavior for waking up BSP. To
 946 * void the boot-strap code, wake up CPU0 by NMI instead.
 947 *
 948 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 949 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 950 * We'll change this code in the future to wake up hard offlined CPU0 if
 951 * real platform and request are available.
 952 */
 953static int
 954wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
 955	       int *cpu0_nmi_registered)
 956{
 957	int id;
 958	int boot_error;
 959
 960	preempt_disable();
 961
 962	/*
 963	 * Wake up AP by INIT, INIT, STARTUP sequence.
 964	 */
 965	if (cpu) {
 966		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
 967		goto out;
 968	}
 969
 970	/*
 971	 * Wake up BSP by nmi.
 972	 *
 973	 * Register a NMI handler to help wake up CPU0.
 974	 */
 975	boot_error = register_nmi_handler(NMI_LOCAL,
 976					  wakeup_cpu0_nmi, 0, "wake_cpu0");
 977
 978	if (!boot_error) {
 979		enable_start_cpu0 = 1;
 980		*cpu0_nmi_registered = 1;
 981		if (apic->dest_logical == APIC_DEST_LOGICAL)
 982			id = cpu0_logical_apicid;
 983		else
 984			id = apicid;
 985		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
 986	}
 987
 988out:
 989	preempt_enable();
 990
 991	return boot_error;
 992}
 993
 994int common_cpu_up(unsigned int cpu, struct task_struct *idle)
 995{
 996	int ret;
 997
 998	/* Just in case we booted with a single CPU. */
 999	alternatives_enable_smp();
1000
1001	per_cpu(current_task, cpu) = idle;
1002
1003	/* Initialize the interrupt stack(s) */
1004	ret = irq_init_percpu_irqstack(cpu);
1005	if (ret)
1006		return ret;
1007
1008#ifdef CONFIG_X86_32
1009	/* Stack for startup_32 can be just as for start_secondary onwards */
1010	per_cpu(cpu_current_top_of_stack, cpu) = task_top_of_stack(idle);
1011#else
1012	initial_gs = per_cpu_offset(cpu);
1013#endif
1014	return 0;
1015}
1016
1017/*
1018 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
1019 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
1020 * Returns zero if CPU booted OK, else error code from
1021 * ->wakeup_secondary_cpu.
1022 */
1023static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
1024		       int *cpu0_nmi_registered)
1025{
 
 
1026	/* start_ip had better be page-aligned! */
1027	unsigned long start_ip = real_mode_header->trampoline_start;
1028
1029	unsigned long boot_error = 0;
1030	unsigned long timeout;
 
1031
1032	idle->thread.sp = (unsigned long)task_pt_regs(idle);
1033	early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
1034	initial_code = (unsigned long)start_secondary;
1035	initial_stack  = idle->thread.sp;
1036
1037	/* Enable the espfix hack for this CPU */
1038	init_espfix_ap(cpu);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1039
1040	/* So we see what's up */
1041	announce_cpu(cpu, apicid);
1042
1043	/*
1044	 * This grunge runs the startup process for
1045	 * the targeted processor.
1046	 */
1047
1048	if (x86_platform.legacy.warm_reset) {
 
 
1049
1050		pr_debug("Setting warm reset code and vector.\n");
1051
1052		smpboot_setup_warm_reset_vector(start_ip);
1053		/*
1054		 * Be paranoid about clearing APIC errors.
1055		*/
1056		if (APIC_INTEGRATED(boot_cpu_apic_version)) {
1057			apic_write(APIC_ESR, 0);
1058			apic_read(APIC_ESR);
1059		}
1060	}
1061
1062	/*
1063	 * AP might wait on cpu_callout_mask in cpu_init() with
1064	 * cpu_initialized_mask set if previous attempt to online
1065	 * it timed-out. Clear cpu_initialized_mask so that after
1066	 * INIT/SIPI it could start with a clean state.
1067	 */
1068	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1069	smp_mb();
1070
1071	/*
1072	 * Wake up a CPU in difference cases:
1073	 * - Use the method in the APIC driver if it's defined
1074	 * Otherwise,
1075	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1076	 */
1077	if (apic->wakeup_secondary_cpu)
1078		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
1079	else
1080		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
1081						     cpu0_nmi_registered);
1082
1083	if (!boot_error) {
1084		/*
1085		 * Wait 10s total for first sign of life from AP
1086		 */
1087		boot_error = -1;
1088		timeout = jiffies + 10*HZ;
1089		while (time_before(jiffies, timeout)) {
1090			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
1091				/*
1092				 * Tell AP to proceed with initialization
1093				 */
1094				cpumask_set_cpu(cpu, cpu_callout_mask);
1095				boot_error = 0;
1096				break;
1097			}
1098			schedule();
1099		}
1100	}
1101
1102	if (!boot_error) {
1103		/*
1104		 * Wait till AP completes initial initialization
1105		 */
1106		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
 
 
 
1107			/*
1108			 * Allow other tasks to run while we wait for the
1109			 * AP to come online. This also gives a chance
1110			 * for the MTRR work(triggered by the AP coming online)
1111			 * to be completed in the stop machine context.
1112			 */
1113			schedule();
1114		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1115	}
1116
1117	if (x86_platform.legacy.warm_reset) {
 
 
 
1118		/*
1119		 * Cleanup possible dangling ends...
1120		 */
1121		smpboot_restore_warm_reset_vector();
1122	}
 
 
 
 
 
 
1123
1124	return boot_error;
1125}
1126
1127int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1128{
1129	int apicid = apic->cpu_present_to_apicid(cpu);
1130	int cpu0_nmi_registered = 0;
1131	unsigned long flags;
1132	int err, ret = 0;
1133
1134	lockdep_assert_irqs_enabled();
1135
1136	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1137
1138	if (apicid == BAD_APICID ||
1139	    !physid_isset(apicid, phys_cpu_present_map) ||
1140	    !apic->apic_id_valid(apicid)) {
1141		pr_err("%s: bad cpu %d\n", __func__, cpu);
1142		return -EINVAL;
1143	}
1144
1145	/*
1146	 * Already booted CPU?
1147	 */
1148	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1149		pr_debug("do_boot_cpu %d Already started\n", cpu);
1150		return -ENOSYS;
1151	}
1152
1153	/*
1154	 * Save current MTRR state in case it was changed since early boot
1155	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
1156	 */
1157	mtrr_save_state();
1158
1159	/* x86 CPUs take themselves offline, so delayed offline is OK. */
1160	err = cpu_check_up_prepare(cpu);
1161	if (err && err != -EBUSY)
1162		return err;
1163
1164	/* the FPU context is blank, nobody can own it */
1165	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
1166
1167	err = common_cpu_up(cpu, tidle);
1168	if (err)
1169		return err;
1170
1171	err = do_boot_cpu(apicid, cpu, tidle, &cpu0_nmi_registered);
1172	if (err) {
1173		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1174		ret = -EIO;
1175		goto unreg_nmi;
1176	}
1177
1178	/*
1179	 * Check TSC synchronization with the AP (keep irqs disabled
1180	 * while doing so):
1181	 */
1182	local_irq_save(flags);
1183	check_tsc_sync_source(cpu);
1184	local_irq_restore(flags);
1185
1186	while (!cpu_online(cpu)) {
1187		cpu_relax();
1188		touch_nmi_watchdog();
1189	}
1190
1191unreg_nmi:
1192	/*
1193	 * Clean up the nmi handler. Do this after the callin and callout sync
1194	 * to avoid impact of possible long unregister time.
1195	 */
1196	if (cpu0_nmi_registered)
1197		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");
1198
1199	return ret;
1200}
1201
1202/**
1203 * arch_disable_smp_support() - disables SMP support for x86 at runtime
1204 */
1205void arch_disable_smp_support(void)
1206{
1207	disable_ioapic_support();
1208}
1209
1210/*
1211 * Fall back to non SMP mode after errors.
1212 *
1213 * RED-PEN audit/test this more. I bet there is more state messed up here.
1214 */
1215static __init void disable_smp(void)
1216{
1217	pr_info("SMP disabled\n");
1218
1219	disable_ioapic_support();
1220
1221	init_cpu_present(cpumask_of(0));
1222	init_cpu_possible(cpumask_of(0));
 
1223
1224	if (smp_found_config)
1225		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1226	else
1227		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1228	cpumask_set_cpu(0, topology_sibling_cpumask(0));
1229	cpumask_set_cpu(0, topology_core_cpumask(0));
1230	cpumask_set_cpu(0, topology_die_cpumask(0));
1231}
1232
1233/*
1234 * Various sanity checks.
1235 */
1236static void __init smp_sanity_check(void)
1237{
1238	preempt_disable();
1239
1240#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1241	if (def_to_bigsmp && nr_cpu_ids > 8) {
1242		unsigned int cpu;
1243		unsigned nr;
1244
1245		pr_warn("More than 8 CPUs detected - skipping them\n"
1246			"Use CONFIG_X86_BIGSMP\n");
1247
1248		nr = 0;
1249		for_each_present_cpu(cpu) {
1250			if (nr >= 8)
1251				set_cpu_present(cpu, false);
1252			nr++;
1253		}
1254
1255		nr = 0;
1256		for_each_possible_cpu(cpu) {
1257			if (nr >= 8)
1258				set_cpu_possible(cpu, false);
1259			nr++;
1260		}
1261
1262		nr_cpu_ids = 8;
1263	}
1264#endif
1265
1266	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1267		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
1268			hard_smp_processor_id());
1269
1270		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1271	}
1272
1273	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
1274	 * Should not be necessary because the MP table should list the boot
1275	 * CPU too, but we do it for the sake of robustness anyway.
1276	 */
1277	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1278		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
1279			  boot_cpu_physical_apicid);
1280		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1281	}
1282	preempt_enable();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1283}
1284
1285static void __init smp_cpu_index_default(void)
1286{
1287	int i;
1288	struct cpuinfo_x86 *c;
1289
1290	for_each_possible_cpu(i) {
1291		c = &cpu_data(i);
1292		/* mark all to hotplug */
1293		c->cpu_index = nr_cpu_ids;
1294	}
1295}
1296
1297static void __init smp_get_logical_apicid(void)
1298{
1299	if (x2apic_mode)
1300		cpu0_logical_apicid = apic_read(APIC_LDR);
1301	else
1302		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
1303}
1304
1305/*
1306 * Prepare for SMP bootup.
1307 * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1308 *            for common interface support.
1309 */
1310void __init native_smp_prepare_cpus(unsigned int max_cpus)
1311{
1312	unsigned int i;
1313
 
1314	smp_cpu_index_default();
1315
1316	/*
1317	 * Setup boot CPU information
1318	 */
1319	smp_store_boot_cpu_info(); /* Final full version of the data */
1320	cpumask_copy(cpu_callin_mask, cpumask_of(0));
1321	mb();
1322
 
1323	for_each_possible_cpu(i) {
1324		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
1325		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1326		zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
1327		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1328	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1329
1330	/*
1331	 * Set 'default' x86 topology, this matches default_topology() in that
1332	 * it has NUMA nodes as a topology level. See also
1333	 * native_smp_cpus_done().
1334	 *
1335	 * Must be done before set_cpus_sibling_map() is ran.
1336	 */
1337	set_sched_topology(x86_topology);
1338
1339	set_cpu_sibling_map(0);
 
 
 
1340
1341	smp_sanity_check();
 
 
 
 
1342
1343	switch (apic_intr_mode) {
1344	case APIC_PIC:
1345	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1346		disable_smp();
1347		return;
1348	case APIC_SYMMETRIC_IO_NO_ROUTING:
1349		disable_smp();
1350		/* Setup local timer */
1351		x86_init.timers.setup_percpu_clockev();
1352		return;
1353	case APIC_VIRTUAL_WIRE:
1354	case APIC_SYMMETRIC_IO:
1355		break;
1356	}
1357
1358	/* Setup local timer */
1359	x86_init.timers.setup_percpu_clockev();
1360
1361	smp_get_logical_apicid();
 
 
 
1362
1363	pr_info("CPU0: ");
1364	print_cpu_info(&cpu_data(0));
 
1365
1366	uv_system_init();
 
1367
1368	set_mtrr_aps_delayed_init();
1369
1370	smp_quirk_init_udelay();
1371
1372	speculative_store_bypass_ht_init();
1373}
1374
1375void arch_enable_nonboot_cpus_begin(void)
1376{
1377	set_mtrr_aps_delayed_init();
1378}
1379
1380void arch_enable_nonboot_cpus_end(void)
1381{
1382	mtrr_aps_init();
1383}
1384
1385/*
1386 * Early setup to make printk work.
1387 */
1388void __init native_smp_prepare_boot_cpu(void)
1389{
1390	int me = smp_processor_id();
1391	switch_to_new_gdt(me);
1392	/* already set me in cpu_online_mask in boot_cpu_init() */
1393	cpumask_set_cpu(me, cpu_callout_mask);
1394	cpu_set_state_online(me);
1395	native_pv_lock_init();
1396}
1397
1398void __init calculate_max_logical_packages(void)
1399{
1400	int ncpus;
1401
1402	/*
1403	 * Today neither Intel nor AMD support heterogenous systems so
1404	 * extrapolate the boot cpu's data to all packages.
1405	 */
1406	ncpus = cpu_data(0).booted_cores * topology_max_smt_threads();
1407	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
1408	pr_info("Max logical packages: %u\n", __max_logical_packages);
1409}
1410
1411void __init native_smp_cpus_done(unsigned int max_cpus)
1412{
1413	pr_debug("Boot done\n");
1414
1415	calculate_max_logical_packages();
1416
1417	if (x86_has_numa_in_package)
1418		set_sched_topology(x86_numa_in_package_topology);
1419
1420	nmi_selftest();
1421	impress_friends();
 
 
 
1422	mtrr_aps_init();
1423}
1424
1425static int __initdata setup_possible_cpus = -1;
1426static int __init _setup_possible_cpus(char *str)
1427{
1428	get_option(&str, &setup_possible_cpus);
1429	return 0;
1430}
1431early_param("possible_cpus", _setup_possible_cpus);
1432
1433
1434/*
1435 * cpu_possible_mask should be static, it cannot change as cpu's
1436 * are onlined, or offlined. The reason is per-cpu data-structures
1437 * are allocated by some modules at init time, and dont expect to
1438 * do this dynamically on cpu arrival/departure.
1439 * cpu_present_mask on the other hand can change dynamically.
1440 * In case when cpu_hotplug is not compiled, then we resort to current
1441 * behaviour, which is cpu_possible == cpu_present.
1442 * - Ashok Raj
1443 *
1444 * Three ways to find out the number of additional hotplug CPUs:
1445 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1446 * - The user can overwrite it with possible_cpus=NUM
1447 * - Otherwise don't reserve additional CPUs.
1448 * We do this because additional CPUs waste a lot of memory.
1449 * -AK
1450 */
1451__init void prefill_possible_map(void)
1452{
1453	int i, possible;
1454
1455	/* No boot processor was found in mptable or ACPI MADT */
1456	if (!num_processors) {
1457		if (boot_cpu_has(X86_FEATURE_APIC)) {
1458			int apicid = boot_cpu_physical_apicid;
1459			int cpu = hard_smp_processor_id();
1460
1461			pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu);
1462
1463			/* Make sure boot cpu is enumerated */
1464			if (apic->cpu_present_to_apicid(0) == BAD_APICID &&
1465			    apic->apic_id_valid(apicid))
1466				generic_processor_info(apicid, boot_cpu_apic_version);
1467		}
1468
1469		if (!num_processors)
1470			num_processors = 1;
1471	}
1472
1473	i = setup_max_cpus ?: 1;
1474	if (setup_possible_cpus == -1) {
1475		possible = num_processors;
1476#ifdef CONFIG_HOTPLUG_CPU
1477		if (setup_max_cpus)
1478			possible += disabled_cpus;
1479#else
1480		if (possible > i)
1481			possible = i;
1482#endif
1483	} else
1484		possible = setup_possible_cpus;
1485
1486	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
1487
1488	/* nr_cpu_ids could be reduced via nr_cpus= */
1489	if (possible > nr_cpu_ids) {
1490		pr_warn("%d Processors exceeds NR_CPUS limit of %u\n",
1491			possible, nr_cpu_ids);
1492		possible = nr_cpu_ids;
1493	}
1494
1495#ifdef CONFIG_HOTPLUG_CPU
1496	if (!setup_max_cpus)
1497#endif
1498	if (possible > i) {
1499		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1500			possible, setup_max_cpus);
1501		possible = i;
1502	}
1503
1504	nr_cpu_ids = possible;
1505
1506	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1507		possible, max_t(int, possible - num_processors, 0));
1508
1509	reset_cpu_possible_mask();
1510
1511	for (i = 0; i < possible; i++)
1512		set_cpu_possible(i, true);
 
 
 
 
1513}
1514
1515#ifdef CONFIG_HOTPLUG_CPU
1516
1517/* Recompute SMT state for all CPUs on offline */
1518static void recompute_smt_state(void)
1519{
1520	int max_threads, cpu;
1521
1522	max_threads = 0;
1523	for_each_online_cpu (cpu) {
1524		int threads = cpumask_weight(topology_sibling_cpumask(cpu));
1525
1526		if (threads > max_threads)
1527			max_threads = threads;
1528	}
1529	__max_smt_threads = max_threads;
1530}
1531
1532static void remove_siblinginfo(int cpu)
1533{
1534	int sibling;
1535	struct cpuinfo_x86 *c = &cpu_data(cpu);
1536
1537	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
1538		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1539		/*/
1540		 * last thread sibling in this cpu core going down
1541		 */
1542		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1543			cpu_data(sibling).booted_cores--;
1544	}
1545
1546	for_each_cpu(sibling, topology_die_cpumask(cpu))
1547		cpumask_clear_cpu(cpu, topology_die_cpumask(sibling));
1548	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
1549		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1550	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
1551		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
1552	cpumask_clear(cpu_llc_shared_mask(cpu));
1553	cpumask_clear(topology_sibling_cpumask(cpu));
1554	cpumask_clear(topology_core_cpumask(cpu));
1555	cpumask_clear(topology_die_cpumask(cpu));
1556	c->cpu_core_id = 0;
1557	c->booted_cores = 0;
1558	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1559	recompute_smt_state();
1560}
1561
1562static void remove_cpu_from_maps(int cpu)
1563{
1564	set_cpu_online(cpu, false);
1565	cpumask_clear_cpu(cpu, cpu_callout_mask);
1566	cpumask_clear_cpu(cpu, cpu_callin_mask);
1567	/* was set by cpu_init() */
1568	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1569	numa_remove_cpu(cpu);
1570}
1571
1572void cpu_disable_common(void)
1573{
1574	int cpu = smp_processor_id();
1575
1576	remove_siblinginfo(cpu);
1577
1578	/* It's now safe to remove this processor from the online map */
1579	lock_vector_lock();
1580	remove_cpu_from_maps(cpu);
1581	unlock_vector_lock();
1582	fixup_irqs();
1583	lapic_offline();
1584}
1585
1586int native_cpu_disable(void)
1587{
1588	int ret;
1589
1590	ret = lapic_can_unplug_cpu();
1591	if (ret)
1592		return ret;
1593
1594	/*
1595	 * Disable the local APIC. Otherwise IPI broadcasts will reach
1596	 * it. It still responds normally to INIT, NMI, SMI, and SIPI
1597	 * messages.
1598	 */
1599	apic_soft_disable();
1600	cpu_disable_common();
1601
 
1602	return 0;
1603}
1604
1605int common_cpu_die(unsigned int cpu)
1606{
1607	int ret = 0;
1608
1609	/* We don't do anything here: idle task is faking death itself. */
 
1610
1611	/* They ack this in play_dead() by setting CPU_DEAD */
1612	if (cpu_wait_death(cpu, 5)) {
1613		if (system_state == SYSTEM_RUNNING)
1614			pr_info("CPU %u is now offline\n", cpu);
1615	} else {
1616		pr_err("CPU %u didn't die...\n", cpu);
1617		ret = -1;
 
1618	}
1619
1620	return ret;
1621}
1622
1623void native_cpu_die(unsigned int cpu)
1624{
1625	common_cpu_die(cpu);
1626}
1627
1628void play_dead_common(void)
1629{
1630	idle_task_exit();
 
 
1631
 
1632	/* Ack it */
1633	(void)cpu_report_death();
1634
1635	/*
1636	 * With physical CPU hotplug, we should halt the cpu
1637	 */
1638	local_irq_disable();
1639}
1640
1641static bool wakeup_cpu0(void)
1642{
1643	if (smp_processor_id() == 0 && enable_start_cpu0)
1644		return true;
1645
1646	return false;
1647}
1648
1649/*
1650 * We need to flush the caches before going to sleep, lest we have
1651 * dirty data in our caches when we come back up.
1652 */
1653static inline void mwait_play_dead(void)
1654{
1655	unsigned int eax, ebx, ecx, edx;
1656	unsigned int highest_cstate = 0;
1657	unsigned int highest_subcstate = 0;
1658	void *mwait_ptr;
1659	int i;
1660
1661	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1662	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
1663		return;
1664	if (!this_cpu_has(X86_FEATURE_MWAIT))
1665		return;
1666	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1667		return;
1668	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1669		return;
1670
1671	eax = CPUID_MWAIT_LEAF;
1672	ecx = 0;
1673	native_cpuid(&eax, &ebx, &ecx, &edx);
1674
1675	/*
1676	 * eax will be 0 if EDX enumeration is not valid.
1677	 * Initialized below to cstate, sub_cstate value when EDX is valid.
1678	 */
1679	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
1680		eax = 0;
1681	} else {
1682		edx >>= MWAIT_SUBSTATE_SIZE;
1683		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
1684			if (edx & MWAIT_SUBSTATE_MASK) {
1685				highest_cstate = i;
1686				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
1687			}
1688		}
1689		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
1690			(highest_subcstate - 1);
1691	}
1692
1693	/*
1694	 * This should be a memory location in a cache line which is
1695	 * unlikely to be touched by other processors.  The actual
1696	 * content is immaterial as it is not actually modified in any way.
1697	 */
1698	mwait_ptr = &current_thread_info()->flags;
1699
1700	wbinvd();
1701
1702	while (1) {
1703		/*
1704		 * The CLFLUSH is a workaround for erratum AAI65 for
1705		 * the Xeon 7400 series.  It's not clear it is actually
1706		 * needed, but it should be harmless in either case.
1707		 * The WBINVD is insufficient due to the spurious-wakeup
1708		 * case where we return around the loop.
1709		 */
1710		mb();
1711		clflush(mwait_ptr);
1712		mb();
1713		__monitor(mwait_ptr, 0, 0);
1714		mb();
1715		__mwait(eax, 0);
1716		/*
1717		 * If NMI wants to wake up CPU0, start CPU0.
1718		 */
1719		if (wakeup_cpu0())
1720			start_cpu0();
1721	}
1722}
1723
1724void hlt_play_dead(void)
1725{
1726	if (__this_cpu_read(cpu_info.x86) >= 4)
1727		wbinvd();
1728
1729	while (1) {
1730		native_halt();
1731		/*
1732		 * If NMI wants to wake up CPU0, start CPU0.
1733		 */
1734		if (wakeup_cpu0())
1735			start_cpu0();
1736	}
1737}
1738
1739void native_play_dead(void)
1740{
1741	play_dead_common();
1742	tboot_shutdown(TB_SHUTDOWN_WFS);
1743
1744	mwait_play_dead();	/* Only returns on failure */
1745	if (cpuidle_play_dead())
1746		hlt_play_dead();
1747}
1748
1749#else /* ... !CONFIG_HOTPLUG_CPU */
1750int native_cpu_disable(void)
1751{
1752	return -ENOSYS;
1753}
1754
1755void native_cpu_die(unsigned int cpu)
1756{
1757	/* We said "no" in __cpu_disable */
1758	BUG();
1759}
1760
1761void native_play_dead(void)
1762{
1763	BUG();
1764}
1765
1766#endif