1// SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3 *	x86 SMP booting functions
   4 *
   5 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
   6 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
   7 *	Copyright 2001 Andi Kleen, SuSE Labs.
   8 *
   9 *	Much of the core SMP work is based on previous work by Thomas Radke, to
  10 *	whom a great many thanks are extended.
  11 *
  12 *	Thanks to Intel for making available several different Pentium,
  13 *	Pentium Pro and Pentium-II/Xeon MP machines.
  14 *	Original development of Linux SMP code supported by Caldera.
  15 *
 
 
 
  16 *	Fixes
  17 *		Felix Koop	:	NR_CPUS used properly
  18 *		Jose Renau	:	Handle single CPU case.
  19 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
  20 *		Greg Wright	:	Fix for kernel stacks panic.
  21 *		Erich Boleyn	:	MP v1.4 and additional changes.
  22 *	Matthias Sattler	:	Changes for 2.1 kernel map.
  23 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
  24 *	Michael Chastain	:	Change trampoline.S to gnu as.
  25 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
  26 *		Ingo Molnar	:	Added APIC timers, based on code
  27 *					from Jose Renau
  28 *		Ingo Molnar	:	various cleanups and rewrites
  29 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
  30 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
  31 *	Andi Kleen		:	Changed for SMP boot into long mode.
  32 *		Martin J. Bligh	: 	Added support for multi-quad systems
  33 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
  34 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
  35 *      Andi Kleen              :       Converted to new state machine.
  36 *	Ashok Raj		: 	CPU hotplug support
  37 *	Glauber Costa		:	i386 and x86_64 integration
  38 */
  39
  40#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  41
  42#include <linux/init.h>
  43#include <linux/smp.h>
  44#include <linux/export.h>
  45#include <linux/sched.h>
  46#include <linux/sched/topology.h>
  47#include <linux/sched/hotplug.h>
  48#include <linux/sched/task_stack.h>
  49#include <linux/percpu.h>
  50#include <linux/memblock.h>
  51#include <linux/err.h>
  52#include <linux/nmi.h>
  53#include <linux/tboot.h>
  54#include <linux/gfp.h>
  55#include <linux/cpuidle.h>
  56#include <linux/numa.h>
  57#include <linux/pgtable.h>
  58#include <linux/overflow.h>
  59#include <linux/stackprotector.h>
 
  60
  61#include <asm/acpi.h>
  62#include <asm/cacheinfo.h>
  63#include <asm/desc.h>
  64#include <asm/nmi.h>
  65#include <asm/irq.h>
  66#include <asm/realmode.h>
 
  67#include <asm/cpu.h>
  68#include <asm/numa.h>
 
  69#include <asm/tlbflush.h>
  70#include <asm/mtrr.h>
  71#include <asm/mwait.h>
  72#include <asm/apic.h>
  73#include <asm/io_apic.h>
  74#include <asm/fpu/api.h>
  75#include <asm/setup.h>
  76#include <asm/uv/uv.h>
  77#include <linux/mc146818rtc.h>
  78#include <asm/i8259.h>
  79#include <asm/misc.h>
  80#include <asm/qspinlock.h>
  81#include <asm/intel-family.h>
  82#include <asm/cpu_device_id.h>
  83#include <asm/spec-ctrl.h>
  84#include <asm/hw_irq.h>
  85#include <asm/stackprotector.h>
  86#include <asm/sev.h>
  87
  88/* representing HT siblings of each logical CPU */
  89DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
  90EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  91
  92/* representing HT and core siblings of each logical CPU */
  93DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
  94EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  95
  96/* representing HT, core, and die siblings of each logical CPU */
  97DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map);
  98EXPORT_PER_CPU_SYMBOL(cpu_die_map);
 
 
 
 
 
 
 
 
 
  99
 100/* Per CPU bogomips and other parameters */
 101DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
 102EXPORT_PER_CPU_SYMBOL(cpu_info);
 
 
 103
 104/* Logical package management. We might want to allocate that dynamically */
 105unsigned int __max_logical_packages __read_mostly;
 106EXPORT_SYMBOL(__max_logical_packages);
 107static unsigned int logical_packages __read_mostly;
 108static unsigned int logical_die __read_mostly;
 109
 110/* Maximum number of SMT threads on any online core */
 111int __read_mostly __max_smt_threads = 1;
 
 
 112
 113/* Flag to indicate if a complete sched domain rebuild is required */
 114bool x86_topology_update;
 
 
 
 
 
 115
 116int arch_update_cpu_topology(void)
 117{
 118	int retval = x86_topology_update;
 119
 120	x86_topology_update = false;
 121	return retval;
 122}
 123
 124static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
 125{
 126	unsigned long flags;
 127
 128	spin_lock_irqsave(&rtc_lock, flags);
 129	CMOS_WRITE(0xa, 0xf);
 130	spin_unlock_irqrestore(&rtc_lock, flags);
 131	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
 132							start_eip >> 4;
 133	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
 134							start_eip & 0xf;
 135}
 136
 137static inline void smpboot_restore_warm_reset_vector(void)
 138{
 139	unsigned long flags;
 140
 141	/*
 142	 * Paranoid:  Set warm reset code and vector here back
 143	 * to default values.
 144	 */
 145	spin_lock_irqsave(&rtc_lock, flags);
 146	CMOS_WRITE(0, 0xf);
 147	spin_unlock_irqrestore(&rtc_lock, flags);
 148
 149	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
 150}
 151
 152/*
 153 * Report back to the Boot Processor during boot time or to the caller processor
 154 * during CPU online.
 155 */
 156static void smp_callin(void)
 157{
 158	int cpuid;
 
 159
 160	/*
 161	 * If waken up by an INIT in an 82489DX configuration
 162	 * cpu_callout_mask guarantees we don't get here before
 163	 * an INIT_deassert IPI reaches our local APIC, so it is
 164	 * now safe to touch our local APIC.
 165	 */
 
 
 
 
 
 
 
 166	cpuid = smp_processor_id();
 
 
 
 
 
 167
 168	/*
 169	 * the boot CPU has finished the init stage and is spinning
 170	 * on callin_map until we finish. We are free to set up this
 171	 * CPU, first the APIC. (this is probably redundant on most
 172	 * boards)
 
 173	 */
 174	apic_ap_setup();
 175
 176	/*
 177	 * Save our processor parameters. Note: this information
 178	 * is needed for clock calibration.
 179	 */
 180	smp_store_cpu_info(cpuid);
 
 
 
 
 
 
 
 
 
 
 
 
 
 181
 182	/*
 183	 * The topology information must be up to date before
 184	 * calibrate_delay() and notify_cpu_starting().
 
 
 185	 */
 186	set_cpu_sibling_map(raw_smp_processor_id());
 187
 188	ap_init_aperfmperf();
 
 
 
 
 189
 190	/*
 
 
 
 
 191	 * Get our bogomips.
 192	 * Update loops_per_jiffy in cpu_data. Previous call to
 193	 * smp_store_cpu_info() stored a value that is close but not as
 194	 * accurate as the value just calculated.
 195	 */
 
 196	calibrate_delay();
 197	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
 198	pr_debug("Stack at about %p\n", &cpuid);
 199
 
 
 
 
 
 
 
 
 
 
 200	wmb();
 201
 202	notify_cpu_starting(cpuid);
 203
 204	/*
 205	 * Allow the master to continue.
 206	 */
 207	cpumask_set_cpu(cpuid, cpu_callin_mask);
 208}
 209
 210static int cpu0_logical_apicid;
 211static int enable_start_cpu0;
 212/*
 213 * Activate a secondary processor.
 214 */
 215static void notrace start_secondary(void *unused)
 216{
 217	/*
 218	 * Don't put *anything* except direct CPU state initialization
 219	 * before cpu_init(), SMP booting is too fragile that we want to
 220	 * limit the things done here to the most necessary things.
 221	 */
 222	cr4_init();
 
 
 223
 224#ifdef CONFIG_X86_32
 225	/* switch away from the initial page table */
 226	load_cr3(swapper_pg_dir);
 227	__flush_tlb_all();
 228#endif
 229	cpu_init_secondary();
 230	rcu_cpu_starting(raw_smp_processor_id());
 231	x86_cpuinit.early_percpu_clock_init();
 232	smp_callin();
 233
 234	enable_start_cpu0 = 0;
 235
 236	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 237	barrier();
 238	/*
 239	 * Check TSC synchronization with the boot CPU:
 240	 */
 241	check_tsc_sync_target();
 242
 243	speculative_store_bypass_ht_init();
 244
 245	/*
 246	 * Lock vector_lock, set CPU online and bring the vector
 247	 * allocator online. Online must be set with vector_lock held
 248	 * to prevent a concurrent irq setup/teardown from seeing a
 249	 * half valid vector space.
 
 
 
 
 
 
 250	 */
 
 251	lock_vector_lock();
 252	set_cpu_online(smp_processor_id(), true);
 253	lapic_online();
 254	unlock_vector_lock();
 255	cpu_set_state_online(smp_processor_id());
 
 256	x86_platform.nmi_init();
 257
 
 
 
 
 
 
 
 
 
 
 
 
 
 258	/* enable local interrupts */
 259	local_irq_enable();
 260
 
 
 
 261	x86_cpuinit.setup_percpu_clockev();
 262
 263	wmb();
 264	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 265}
 266
 267/**
 268 * topology_is_primary_thread - Check whether CPU is the primary SMT thread
 269 * @cpu:	CPU to check
 270 */
 271bool topology_is_primary_thread(unsigned int cpu)
 272{
 273	return apic_id_is_primary_thread(per_cpu(x86_cpu_to_apicid, cpu));
 274}
 275
 276/**
 277 * topology_smt_supported - Check whether SMT is supported by the CPUs
 278 */
 279bool topology_smt_supported(void)
 280{
 281	return smp_num_siblings > 1;
 282}
 283
 284/**
 285 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 286 *
 287 * Returns logical package id or -1 if not found
 288 */
 289int topology_phys_to_logical_pkg(unsigned int phys_pkg)
 290{
 291	int cpu;
 292
 293	for_each_possible_cpu(cpu) {
 294		struct cpuinfo_x86 *c = &cpu_data(cpu);
 295
 296		if (c->initialized && c->phys_proc_id == phys_pkg)
 297			return c->logical_proc_id;
 298	}
 299	return -1;
 300}
 301EXPORT_SYMBOL(topology_phys_to_logical_pkg);
 302/**
 303 * topology_phys_to_logical_die - Map a physical die id to logical
 304 *
 305 * Returns logical die id or -1 if not found
 306 */
 307int topology_phys_to_logical_die(unsigned int die_id, unsigned int cur_cpu)
 308{
 309	int cpu;
 310	int proc_id = cpu_data(cur_cpu).phys_proc_id;
 311
 312	for_each_possible_cpu(cpu) {
 313		struct cpuinfo_x86 *c = &cpu_data(cpu);
 314
 315		if (c->initialized && c->cpu_die_id == die_id &&
 316		    c->phys_proc_id == proc_id)
 317			return c->logical_die_id;
 318	}
 319	return -1;
 320}
 321EXPORT_SYMBOL(topology_phys_to_logical_die);
 322
 323/**
 324 * topology_update_package_map - Update the physical to logical package map
 325 * @pkg:	The physical package id as retrieved via CPUID
 326 * @cpu:	The cpu for which this is updated
 327 */
 328int topology_update_package_map(unsigned int pkg, unsigned int cpu)
 329{
 330	int new;
 331
 332	/* Already available somewhere? */
 333	new = topology_phys_to_logical_pkg(pkg);
 334	if (new >= 0)
 335		goto found;
 336
 337	new = logical_packages++;
 338	if (new != pkg) {
 339		pr_info("CPU %u Converting physical %u to logical package %u\n",
 340			cpu, pkg, new);
 341	}
 342found:
 343	cpu_data(cpu).logical_proc_id = new;
 344	return 0;
 345}
 346/**
 347 * topology_update_die_map - Update the physical to logical die map
 348 * @die:	The die id as retrieved via CPUID
 349 * @cpu:	The cpu for which this is updated
 350 */
 351int topology_update_die_map(unsigned int die, unsigned int cpu)
 352{
 353	int new;
 354
 355	/* Already available somewhere? */
 356	new = topology_phys_to_logical_die(die, cpu);
 357	if (new >= 0)
 358		goto found;
 359
 360	new = logical_die++;
 361	if (new != die) {
 362		pr_info("CPU %u Converting physical %u to logical die %u\n",
 363			cpu, die, new);
 364	}
 365found:
 366	cpu_data(cpu).logical_die_id = new;
 367	return 0;
 368}
 369
 370void __init smp_store_boot_cpu_info(void)
 371{
 372	int id = 0; /* CPU 0 */
 373	struct cpuinfo_x86 *c = &cpu_data(id);
 374
 375	*c = boot_cpu_data;
 376	c->cpu_index = id;
 377	topology_update_package_map(c->phys_proc_id, id);
 378	topology_update_die_map(c->cpu_die_id, id);
 379	c->initialized = true;
 380}
 381
 382/*
 383 * The bootstrap kernel entry code has set these up. Save them for
 384 * a given CPU
 385 */
 386void smp_store_cpu_info(int id)
 
 387{
 388	struct cpuinfo_x86 *c = &cpu_data(id);
 389
 390	/* Copy boot_cpu_data only on the first bringup */
 391	if (!c->initialized)
 392		*c = boot_cpu_data;
 393	c->cpu_index = id;
 394	/*
 395	 * During boot time, CPU0 has this setup already. Save the info when
 396	 * bringing up AP or offlined CPU0.
 397	 */
 398	identify_secondary_cpu(c);
 399	c->initialized = true;
 400}
 401
 402static bool
 403topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 404{
 405	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 406
 407	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
 
 
 
 408}
 409
 410static bool
 411topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
 412{
 413	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 414
 415	return !WARN_ONCE(!topology_same_node(c, o),
 416		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
 417		"[node: %d != %d]. Ignoring dependency.\n",
 418		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
 419}
 420
 421#define link_mask(mfunc, c1, c2)					\
 422do {									\
 423	cpumask_set_cpu((c1), mfunc(c2));				\
 424	cpumask_set_cpu((c2), mfunc(c1));				\
 425} while (0)
 426
 427static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 428{
 429	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
 430		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 431
 432		if (c->phys_proc_id == o->phys_proc_id &&
 433		    c->cpu_die_id == o->cpu_die_id &&
 434		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2)) {
 435			if (c->cpu_core_id == o->cpu_core_id)
 436				return topology_sane(c, o, "smt");
 437
 438			if ((c->cu_id != 0xff) &&
 439			    (o->cu_id != 0xff) &&
 440			    (c->cu_id == o->cu_id))
 441				return topology_sane(c, o, "smt");
 
 
 
 
 
 
 
 
 
 442		}
 443
 444	} else if (c->phys_proc_id == o->phys_proc_id &&
 445		   c->cpu_die_id == o->cpu_die_id &&
 446		   c->cpu_core_id == o->cpu_core_id) {
 447		return topology_sane(c, o, "smt");
 448	}
 449
 450	return false;
 451}
 452
 453static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 454{
 455	if (c->phys_proc_id == o->phys_proc_id &&
 456	    c->cpu_die_id == o->cpu_die_id)
 457		return true;
 458	return false;
 459}
 460
 461static bool match_l2c(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 462{
 463	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 464
 465	/* If the arch didn't set up l2c_id, fall back to SMT */
 466	if (per_cpu(cpu_l2c_id, cpu1) == BAD_APICID)
 467		return match_smt(c, o);
 468
 469	/* Do not match if L2 cache id does not match: */
 470	if (per_cpu(cpu_l2c_id, cpu1) != per_cpu(cpu_l2c_id, cpu2))
 471		return false;
 472
 473	return topology_sane(c, o, "l2c");
 474}
 475
 476/*
 477 * Unlike the other levels, we do not enforce keeping a
 478 * multicore group inside a NUMA node.  If this happens, we will
 479 * discard the MC level of the topology later.
 480 */
 481static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 482{
 483	if (c->phys_proc_id == o->phys_proc_id)
 484		return true;
 485	return false;
 486}
 487
 488/*
 489 * Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs.
 490 *
 491 * Any Intel CPU that has multiple nodes per package and does not
 492 * match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology.
 493 *
 494 * When in SNC mode, these CPUs enumerate an LLC that is shared
 495 * by multiple NUMA nodes. The LLC is shared for off-package data
 496 * access but private to the NUMA node (half of the package) for
 497 * on-package access. CPUID (the source of the information about
 498 * the LLC) can only enumerate the cache as shared or unshared,
 499 * but not this particular configuration.
 500 */
 501
 502static const struct x86_cpu_id intel_cod_cpu[] = {
 503	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, 0),	/* COD */
 504	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, 0),	/* COD */
 505	X86_MATCH_INTEL_FAM6_MODEL(ANY, 1),		/* SNC */
 506	{}
 507};
 508
 509static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 510{
 511	const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
 512	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 513	bool intel_snc = id && id->driver_data;
 514
 515	/* Do not match if we do not have a valid APICID for cpu: */
 516	if (per_cpu(cpu_llc_id, cpu1) == BAD_APICID)
 517		return false;
 518
 519	/* Do not match if LLC id does not match: */
 520	if (per_cpu(cpu_llc_id, cpu1) != per_cpu(cpu_llc_id, cpu2))
 521		return false;
 522
 523	/*
 524	 * Allow the SNC topology without warning. Return of false
 525	 * means 'c' does not share the LLC of 'o'. This will be
 526	 * reflected to userspace.
 527	 */
 528	if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc)
 529		return false;
 530
 531	return topology_sane(c, o, "llc");
 532}
 533
 534
 535#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_CLUSTER) || defined(CONFIG_SCHED_MC)
 536static inline int x86_sched_itmt_flags(void)
 537{
 538	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;
 539}
 540
 541#ifdef CONFIG_SCHED_MC
 542static int x86_core_flags(void)
 543{
 544	return cpu_core_flags() | x86_sched_itmt_flags();
 545}
 546#endif
 547#ifdef CONFIG_SCHED_SMT
 548static int x86_smt_flags(void)
 549{
 550	return cpu_smt_flags() | x86_sched_itmt_flags();
 551}
 552#endif
 553#ifdef CONFIG_SCHED_CLUSTER
 554static int x86_cluster_flags(void)
 555{
 556	return cpu_cluster_flags() | x86_sched_itmt_flags();
 557}
 558#endif
 559#endif
 560
 561static struct sched_domain_topology_level x86_numa_in_package_topology[] = {
 562#ifdef CONFIG_SCHED_SMT
 563	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 564#endif
 565#ifdef CONFIG_SCHED_CLUSTER
 566	{ cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS) },
 567#endif
 568#ifdef CONFIG_SCHED_MC
 569	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 570#endif
 571	{ NULL, },
 572};
 573
 574static struct sched_domain_topology_level x86_hybrid_topology[] = {
 575#ifdef CONFIG_SCHED_SMT
 576	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 577#endif
 578#ifdef CONFIG_SCHED_MC
 579	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 580#endif
 581	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
 582	{ NULL, },
 583};
 584
 585static struct sched_domain_topology_level x86_topology[] = {
 586#ifdef CONFIG_SCHED_SMT
 587	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 588#endif
 589#ifdef CONFIG_SCHED_CLUSTER
 590	{ cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS) },
 591#endif
 592#ifdef CONFIG_SCHED_MC
 593	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 594#endif
 595	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
 596	{ NULL, },
 597};
 598
 599/*
 600 * Set if a package/die has multiple NUMA nodes inside.
 601 * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
 602 * Sub-NUMA Clustering have this.
 603 */
 604static bool x86_has_numa_in_package;
 605
 606void set_cpu_sibling_map(int cpu)
 607{
 608	bool has_smt = smp_num_siblings > 1;
 609	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
 610	struct cpuinfo_x86 *c = &cpu_data(cpu);
 611	struct cpuinfo_x86 *o;
 612	int i, threads;
 613
 614	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
 615
 616	if (!has_mp) {
 617		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
 618		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
 619		cpumask_set_cpu(cpu, cpu_l2c_shared_mask(cpu));
 620		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
 621		cpumask_set_cpu(cpu, topology_die_cpumask(cpu));
 622		c->booted_cores = 1;
 623		return;
 624	}
 625
 626	for_each_cpu(i, cpu_sibling_setup_mask) {
 627		o = &cpu_data(i);
 628
 629		if (match_pkg(c, o) && !topology_same_node(c, o))
 630			x86_has_numa_in_package = true;
 631
 632		if ((i == cpu) || (has_smt && match_smt(c, o)))
 633			link_mask(topology_sibling_cpumask, cpu, i);
 634
 635		if ((i == cpu) || (has_mp && match_llc(c, o)))
 636			link_mask(cpu_llc_shared_mask, cpu, i);
 637
 638		if ((i == cpu) || (has_mp && match_l2c(c, o)))
 639			link_mask(cpu_l2c_shared_mask, cpu, i);
 640
 641		if ((i == cpu) || (has_mp && match_die(c, o)))
 642			link_mask(topology_die_cpumask, cpu, i);
 643	}
 644
 645	threads = cpumask_weight(topology_sibling_cpumask(cpu));
 646	if (threads > __max_smt_threads)
 647		__max_smt_threads = threads;
 648
 649	for_each_cpu(i, topology_sibling_cpumask(cpu))
 650		cpu_data(i).smt_active = threads > 1;
 651
 652	/*
 653	 * This needs a separate iteration over the cpus because we rely on all
 654	 * topology_sibling_cpumask links to be set-up.
 655	 */
 656	for_each_cpu(i, cpu_sibling_setup_mask) {
 657		o = &cpu_data(i);
 658
 659		if ((i == cpu) || (has_mp && match_pkg(c, o))) {
 660			link_mask(topology_core_cpumask, cpu, i);
 661
 662			/*
 663			 *  Does this new cpu bringup a new core?
 664			 */
 665			if (threads == 1) {
 666				/*
 667				 * for each core in package, increment
 668				 * the booted_cores for this new cpu
 669				 */
 670				if (cpumask_first(
 671				    topology_sibling_cpumask(i)) == i)
 672					c->booted_cores++;
 673				/*
 674				 * increment the core count for all
 675				 * the other cpus in this package
 676				 */
 677				if (i != cpu)
 678					cpu_data(i).booted_cores++;
 679			} else if (i != cpu && !c->booted_cores)
 680				c->booted_cores = cpu_data(i).booted_cores;
 681		}
 682	}
 683}
 684
 685/* maps the cpu to the sched domain representing multi-core */
 686const struct cpumask *cpu_coregroup_mask(int cpu)
 687{
 688	return cpu_llc_shared_mask(cpu);
 689}
 690
 691const struct cpumask *cpu_clustergroup_mask(int cpu)
 692{
 693	return cpu_l2c_shared_mask(cpu);
 
 
 
 
 694}
 695
 696static void impress_friends(void)
 697{
 698	int cpu;
 699	unsigned long bogosum = 0;
 700	/*
 701	 * Allow the user to impress friends.
 702	 */
 703	pr_debug("Before bogomips\n");
 704	for_each_possible_cpu(cpu)
 705		if (cpumask_test_cpu(cpu, cpu_callout_mask))
 706			bogosum += cpu_data(cpu).loops_per_jiffy;
 707	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
 
 708		num_online_cpus(),
 709		bogosum/(500000/HZ),
 710		(bogosum/(5000/HZ))%100);
 711
 712	pr_debug("Before bogocount - setting activated=1\n");
 713}
 714
 715void __inquire_remote_apic(int apicid)
 716{
 717	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
 718	const char * const names[] = { "ID", "VERSION", "SPIV" };
 719	int timeout;
 720	u32 status;
 721
 722	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
 723
 724	for (i = 0; i < ARRAY_SIZE(regs); i++) {
 725		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
 726
 727		/*
 728		 * Wait for idle.
 729		 */
 730		status = safe_apic_wait_icr_idle();
 731		if (status)
 732			pr_cont("a previous APIC delivery may have failed\n");
 
 733
 734		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
 735
 736		timeout = 0;
 737		do {
 738			udelay(100);
 739			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
 740		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
 741
 742		switch (status) {
 743		case APIC_ICR_RR_VALID:
 744			status = apic_read(APIC_RRR);
 745			pr_cont("%08x\n", status);
 746			break;
 747		default:
 748			pr_cont("failed\n");
 749		}
 750	}
 751}
 752
 753/*
 754 * The Multiprocessor Specification 1.4 (1997) example code suggests
 755 * that there should be a 10ms delay between the BSP asserting INIT
 756 * and de-asserting INIT, when starting a remote processor.
 757 * But that slows boot and resume on modern processors, which include
 758 * many cores and don't require that delay.
 759 *
 760 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
 761 * Modern processor families are quirked to remove the delay entirely.
 762 */
 763#define UDELAY_10MS_DEFAULT 10000
 764
 765static unsigned int init_udelay = UINT_MAX;
 766
 767static int __init cpu_init_udelay(char *str)
 768{
 769	get_option(&str, &init_udelay);
 770
 771	return 0;
 772}
 773early_param("cpu_init_udelay", cpu_init_udelay);
 774
 775static void __init smp_quirk_init_udelay(void)
 776{
 777	/* if cmdline changed it from default, leave it alone */
 778	if (init_udelay != UINT_MAX)
 779		return;
 780
 781	/* if modern processor, use no delay */
 782	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
 783	    ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && (boot_cpu_data.x86 >= 0x18)) ||
 784	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
 785		init_udelay = 0;
 786		return;
 787	}
 788	/* else, use legacy delay */
 789	init_udelay = UDELAY_10MS_DEFAULT;
 790}
 791
 792/*
 793 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 794 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 795 * won't ... remember to clear down the APIC, etc later.
 796 */
 797int
 798wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
 799{
 800	u32 dm = apic->dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
 801	unsigned long send_status, accept_status = 0;
 802	int maxlvt;
 803
 804	/* Target chip */
 805	/* Boot on the stack */
 806	/* Kick the second */
 807	apic_icr_write(APIC_DM_NMI | dm, apicid);
 808
 809	pr_debug("Waiting for send to finish...\n");
 810	send_status = safe_apic_wait_icr_idle();
 811
 812	/*
 813	 * Give the other CPU some time to accept the IPI.
 814	 */
 815	udelay(200);
 816	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 817		maxlvt = lapic_get_maxlvt();
 818		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
 819			apic_write(APIC_ESR, 0);
 820		accept_status = (apic_read(APIC_ESR) & 0xEF);
 821	}
 822	pr_debug("NMI sent\n");
 823
 824	if (send_status)
 825		pr_err("APIC never delivered???\n");
 826	if (accept_status)
 827		pr_err("APIC delivery error (%lx)\n", accept_status);
 828
 829	return (send_status | accept_status);
 830}
 831
 832static int
 833wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
 834{
 835	unsigned long send_status = 0, accept_status = 0;
 836	int maxlvt, num_starts, j;
 837
 838	maxlvt = lapic_get_maxlvt();
 839
 840	/*
 841	 * Be paranoid about clearing APIC errors.
 842	 */
 843	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 844		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 845			apic_write(APIC_ESR, 0);
 846		apic_read(APIC_ESR);
 847	}
 848
 849	pr_debug("Asserting INIT\n");
 850
 851	/*
 852	 * Turn INIT on target chip
 853	 */
 854	/*
 855	 * Send IPI
 856	 */
 857	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
 858		       phys_apicid);
 859
 860	pr_debug("Waiting for send to finish...\n");
 861	send_status = safe_apic_wait_icr_idle();
 862
 863	udelay(init_udelay);
 864
 865	pr_debug("Deasserting INIT\n");
 866
 867	/* Target chip */
 868	/* Send IPI */
 869	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
 870
 871	pr_debug("Waiting for send to finish...\n");
 872	send_status = safe_apic_wait_icr_idle();
 873
 874	mb();
 
 875
 876	/*
 877	 * Should we send STARTUP IPIs ?
 878	 *
 879	 * Determine this based on the APIC version.
 880	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
 881	 */
 882	if (APIC_INTEGRATED(boot_cpu_apic_version))
 883		num_starts = 2;
 884	else
 885		num_starts = 0;
 886
 887	/*
 
 
 
 
 
 
 
 888	 * Run STARTUP IPI loop.
 889	 */
 890	pr_debug("#startup loops: %d\n", num_starts);
 891
 892	for (j = 1; j <= num_starts; j++) {
 893		pr_debug("Sending STARTUP #%d\n", j);
 894		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 895			apic_write(APIC_ESR, 0);
 896		apic_read(APIC_ESR);
 897		pr_debug("After apic_write\n");
 898
 899		/*
 900		 * STARTUP IPI
 901		 */
 902
 903		/* Target chip */
 904		/* Boot on the stack */
 905		/* Kick the second */
 906		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
 907			       phys_apicid);
 908
 909		/*
 910		 * Give the other CPU some time to accept the IPI.
 911		 */
 912		if (init_udelay == 0)
 913			udelay(10);
 914		else
 915			udelay(300);
 916
 917		pr_debug("Startup point 1\n");
 918
 919		pr_debug("Waiting for send to finish...\n");
 920		send_status = safe_apic_wait_icr_idle();
 921
 922		/*
 923		 * Give the other CPU some time to accept the IPI.
 924		 */
 925		if (init_udelay == 0)
 926			udelay(10);
 927		else
 928			udelay(200);
 929
 930		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 931			apic_write(APIC_ESR, 0);
 932		accept_status = (apic_read(APIC_ESR) & 0xEF);
 933		if (send_status || accept_status)
 934			break;
 935	}
 936	pr_debug("After Startup\n");
 937
 938	if (send_status)
 939		pr_err("APIC never delivered???\n");
 940	if (accept_status)
 941		pr_err("APIC delivery error (%lx)\n", accept_status);
 942
 943	return (send_status | accept_status);
 944}
 945
 946/* reduce the number of lines printed when booting a large cpu count system */
 947static void announce_cpu(int cpu, int apicid)
 948{
 949	static int current_node = NUMA_NO_NODE;
 950	int node = early_cpu_to_node(cpu);
 951	static int width, node_width;
 952
 953	if (!width)
 954		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
 
 
 955
 956	if (!node_width)
 957		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */
 
 958
 959	if (cpu == 1)
 960		printk(KERN_INFO "x86: Booting SMP configuration:\n");
 
 
 
 961
 962	if (system_state < SYSTEM_RUNNING) {
 963		if (node != current_node) {
 964			if (current_node > (-1))
 965				pr_cont("\n");
 966			current_node = node;
 967
 968			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
 969			       node_width - num_digits(node), " ", node);
 970		}
 971
 972		/* Add padding for the BSP */
 973		if (cpu == 1)
 974			pr_cont("%*s", width + 1, " ");
 975
 976		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);
 977
 978	} else
 979		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
 980			node, cpu, apicid);
 981}
 982
 983static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
 984{
 985	int cpu;
 986
 987	cpu = smp_processor_id();
 988	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
 989		return NMI_HANDLED;
 990
 991	return NMI_DONE;
 992}
 993
 994/*
 995 * Wake up AP by INIT, INIT, STARTUP sequence.
 996 *
 997 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 998 * boot-strap code which is not a desired behavior for waking up BSP. To
 999 * void the boot-strap code, wake up CPU0 by NMI instead.
1000 *
1001 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
1002 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
1003 * We'll change this code in the future to wake up hard offlined CPU0 if
1004 * real platform and request are available.
1005 */
1006static int
1007wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
1008	       int *cpu0_nmi_registered)
1009{
1010	int id;
1011	int boot_error;
 
 
 
 
 
1012
1013	preempt_disable();
1014
1015	/*
1016	 * Wake up AP by INIT, INIT, STARTUP sequence.
1017	 */
1018	if (cpu) {
1019		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
1020		goto out;
1021	}
1022
1023	/*
1024	 * Wake up BSP by nmi.
1025	 *
1026	 * Register a NMI handler to help wake up CPU0.
1027	 */
1028	boot_error = register_nmi_handler(NMI_LOCAL,
1029					  wakeup_cpu0_nmi, 0, "wake_cpu0");
1030
1031	if (!boot_error) {
1032		enable_start_cpu0 = 1;
1033		*cpu0_nmi_registered = 1;
1034		id = apic->dest_mode_logical ? cpu0_logical_apicid : apicid;
1035		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
1036	}
1037
1038out:
1039	preempt_enable();
1040
1041	return boot_error;
1042}
1043
1044int common_cpu_up(unsigned int cpu, struct task_struct *idle)
1045{
1046	int ret;
1047
1048	/* Just in case we booted with a single CPU. */
1049	alternatives_enable_smp();
1050
1051	per_cpu(pcpu_hot.current_task, cpu) = idle;
1052	cpu_init_stack_canary(cpu, idle);
1053
1054	/* Initialize the interrupt stack(s) */
1055	ret = irq_init_percpu_irqstack(cpu);
1056	if (ret)
1057		return ret;
1058
 
 
 
1059#ifdef CONFIG_X86_32
1060	/* Stack for startup_32 can be just as for start_secondary onwards */
1061	per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle);
1062#else
 
1063	initial_gs = per_cpu_offset(cpu);
 
 
 
1064#endif
1065	return 0;
1066}
1067
1068/*
1069 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
1070 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
1071 * Returns zero if CPU booted OK, else error code from
1072 * ->wakeup_secondary_cpu.
1073 */
1074static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
1075		       int *cpu0_nmi_registered)
1076{
1077	/* start_ip had better be page-aligned! */
1078	unsigned long start_ip = real_mode_header->trampoline_start;
1079
1080	unsigned long boot_error = 0;
1081	unsigned long timeout;
1082
1083#ifdef CONFIG_X86_64
1084	/* If 64-bit wakeup method exists, use the 64-bit mode trampoline IP */
1085	if (apic->wakeup_secondary_cpu_64)
1086		start_ip = real_mode_header->trampoline_start64;
1087#endif
1088	idle->thread.sp = (unsigned long)task_pt_regs(idle);
1089	early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
1090	initial_code = (unsigned long)start_secondary;
1091	initial_stack  = idle->thread.sp;
1092
1093	/* Enable the espfix hack for this CPU */
1094	init_espfix_ap(cpu);
1095
1096	/* So we see what's up */
1097	announce_cpu(cpu, apicid);
1098
1099	/*
1100	 * This grunge runs the startup process for
1101	 * the targeted processor.
1102	 */
1103
1104	if (x86_platform.legacy.warm_reset) {
 
 
 
 
1105
1106		pr_debug("Setting warm reset code and vector.\n");
1107
1108		smpboot_setup_warm_reset_vector(start_ip);
1109		/*
1110		 * Be paranoid about clearing APIC errors.
1111		*/
1112		if (APIC_INTEGRATED(boot_cpu_apic_version)) {
1113			apic_write(APIC_ESR, 0);
1114			apic_read(APIC_ESR);
1115		}
1116	}
1117
1118	/*
1119	 * AP might wait on cpu_callout_mask in cpu_init() with
1120	 * cpu_initialized_mask set if previous attempt to online
1121	 * it timed-out. Clear cpu_initialized_mask so that after
1122	 * INIT/SIPI it could start with a clean state.
1123	 */
1124	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1125	smp_mb();
1126
1127	/*
1128	 * Wake up a CPU in difference cases:
1129	 * - Use a method from the APIC driver if one defined, with wakeup
1130	 *   straight to 64-bit mode preferred over wakeup to RM.
1131	 * Otherwise,
1132	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1133	 */
1134	if (apic->wakeup_secondary_cpu_64)
1135		boot_error = apic->wakeup_secondary_cpu_64(apicid, start_ip);
1136	else if (apic->wakeup_secondary_cpu)
1137		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
1138	else
1139		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
1140						     cpu0_nmi_registered);
1141
1142	if (!boot_error) {
1143		/*
1144		 * Wait 10s total for first sign of life from AP
1145		 */
1146		boot_error = -1;
1147		timeout = jiffies + 10*HZ;
1148		while (time_before(jiffies, timeout)) {
1149			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
1150				/*
1151				 * Tell AP to proceed with initialization
1152				 */
1153				cpumask_set_cpu(cpu, cpu_callout_mask);
1154				boot_error = 0;
1155				break;
1156			}
1157			schedule();
1158		}
1159	}
1160
1161	if (!boot_error) {
1162		/*
1163		 * Wait till AP completes initial initialization
1164		 */
1165		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
 
 
 
1166			/*
1167			 * Allow other tasks to run while we wait for the
1168			 * AP to come online. This also gives a chance
1169			 * for the MTRR work(triggered by the AP coming online)
1170			 * to be completed in the stop machine context.
1171			 */
1172			schedule();
1173		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1174	}
1175
1176	if (x86_platform.legacy.warm_reset) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1177		/*
1178		 * Cleanup possible dangling ends...
1179		 */
1180		smpboot_restore_warm_reset_vector();
1181	}
1182
 
1183	return boot_error;
1184}
1185
1186int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1187{
1188	int apicid = apic->cpu_present_to_apicid(cpu);
1189	int cpu0_nmi_registered = 0;
1190	unsigned long flags;
1191	int err, ret = 0;
1192
1193	lockdep_assert_irqs_enabled();
1194
1195	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1196
1197	if (apicid == BAD_APICID ||
1198	    !physid_isset(apicid, phys_cpu_present_map) ||
1199	    !apic->apic_id_valid(apicid)) {
1200		pr_err("%s: bad cpu %d\n", __func__, cpu);
1201		return -EINVAL;
1202	}
1203
1204	/*
1205	 * Already booted CPU?
1206	 */
1207	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1208		pr_debug("do_boot_cpu %d Already started\n", cpu);
1209		return -ENOSYS;
1210	}
1211
1212	/*
1213	 * Save current MTRR state in case it was changed since early boot
1214	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
1215	 */
1216	mtrr_save_state();
1217
1218	/* x86 CPUs take themselves offline, so delayed offline is OK. */
1219	err = cpu_check_up_prepare(cpu);
1220	if (err && err != -EBUSY)
1221		return err;
1222
1223	/* the FPU context is blank, nobody can own it */
1224	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
1225
1226	err = common_cpu_up(cpu, tidle);
1227	if (err)
1228		return err;
1229
1230	err = do_boot_cpu(apicid, cpu, tidle, &cpu0_nmi_registered);
1231	if (err) {
1232		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1233		ret = -EIO;
1234		goto unreg_nmi;
1235	}
1236
1237	/*
1238	 * Check TSC synchronization with the AP (keep irqs disabled
1239	 * while doing so):
1240	 */
1241	local_irq_save(flags);
1242	check_tsc_sync_source(cpu);
1243	local_irq_restore(flags);
1244
1245	while (!cpu_online(cpu)) {
1246		cpu_relax();
1247		touch_nmi_watchdog();
1248	}
1249
1250unreg_nmi:
1251	/*
1252	 * Clean up the nmi handler. Do this after the callin and callout sync
1253	 * to avoid impact of possible long unregister time.
1254	 */
1255	if (cpu0_nmi_registered)
1256		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");
1257
1258	return ret;
1259}
1260
1261/**
1262 * arch_disable_smp_support() - disables SMP support for x86 at runtime
1263 */
1264void arch_disable_smp_support(void)
1265{
1266	disable_ioapic_support();
1267}
1268
1269/*
1270 * Fall back to non SMP mode after errors.
1271 *
1272 * RED-PEN audit/test this more. I bet there is more state messed up here.
1273 */
1274static __init void disable_smp(void)
1275{
1276	pr_info("SMP disabled\n");
1277
1278	disable_ioapic_support();
1279
1280	init_cpu_present(cpumask_of(0));
1281	init_cpu_possible(cpumask_of(0));
 
1282
1283	if (smp_found_config)
1284		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1285	else
1286		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1287	cpumask_set_cpu(0, topology_sibling_cpumask(0));
1288	cpumask_set_cpu(0, topology_core_cpumask(0));
1289	cpumask_set_cpu(0, topology_die_cpumask(0));
1290}
1291
1292/*
1293 * Various sanity checks.
1294 */
1295static void __init smp_sanity_check(void)
1296{
1297	preempt_disable();
1298
1299#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1300	if (def_to_bigsmp && nr_cpu_ids > 8) {
1301		unsigned int cpu;
1302		unsigned nr;
1303
1304		pr_warn("More than 8 CPUs detected - skipping them\n"
1305			"Use CONFIG_X86_BIGSMP\n");
 
1306
1307		nr = 0;
1308		for_each_present_cpu(cpu) {
1309			if (nr >= 8)
1310				set_cpu_present(cpu, false);
1311			nr++;
1312		}
1313
1314		nr = 0;
1315		for_each_possible_cpu(cpu) {
1316			if (nr >= 8)
1317				set_cpu_possible(cpu, false);
1318			nr++;
1319		}
1320
1321		set_nr_cpu_ids(8);
1322	}
1323#endif
1324
1325	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1326		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
 
1327			hard_smp_processor_id());
1328
1329		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1330	}
1331
1332	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1333	 * Should not be necessary because the MP table should list the boot
1334	 * CPU too, but we do it for the sake of robustness anyway.
1335	 */
1336	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1337		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
1338			  boot_cpu_physical_apicid);
 
1339		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1340	}
1341	preempt_enable();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1342}
1343
1344static void __init smp_cpu_index_default(void)
1345{
1346	int i;
1347	struct cpuinfo_x86 *c;
1348
1349	for_each_possible_cpu(i) {
1350		c = &cpu_data(i);
1351		/* mark all to hotplug */
1352		c->cpu_index = nr_cpu_ids;
1353	}
1354}
1355
1356static void __init smp_get_logical_apicid(void)
1357{
1358	if (x2apic_mode)
1359		cpu0_logical_apicid = apic_read(APIC_LDR);
1360	else
1361		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
1362}
1363
1364void __init smp_prepare_cpus_common(void)
1365{
1366	unsigned int i;
1367
 
1368	smp_cpu_index_default();
1369
1370	/*
1371	 * Setup boot CPU information
1372	 */
1373	smp_store_boot_cpu_info(); /* Final full version of the data */
1374	cpumask_copy(cpu_callin_mask, cpumask_of(0));
1375	mb();
1376
 
1377	for_each_possible_cpu(i) {
1378		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
1379		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1380		zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
1381		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1382		zalloc_cpumask_var(&per_cpu(cpu_l2c_shared_map, i), GFP_KERNEL);
1383	}
1384
1385	/*
1386	 * Set 'default' x86 topology, this matches default_topology() in that
1387	 * it has NUMA nodes as a topology level. See also
1388	 * native_smp_cpus_done().
1389	 *
1390	 * Must be done before set_cpus_sibling_map() is ran.
1391	 */
1392	set_sched_topology(x86_topology);
1393
1394	set_cpu_sibling_map(0);
1395}
1396
1397/*
1398 * Prepare for SMP bootup.
1399 * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1400 *            for common interface support.
1401 */
1402void __init native_smp_prepare_cpus(unsigned int max_cpus)
1403{
1404	smp_prepare_cpus_common();
1405
1406	smp_sanity_check();
1407
1408	switch (apic_intr_mode) {
1409	case APIC_PIC:
1410	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1411		disable_smp();
1412		return;
1413	case APIC_SYMMETRIC_IO_NO_ROUTING:
1414		disable_smp();
1415		/* Setup local timer */
1416		x86_init.timers.setup_percpu_clockev();
1417		return;
1418	case APIC_VIRTUAL_WIRE:
1419	case APIC_SYMMETRIC_IO:
1420		break;
1421	}
1422
1423	/* Setup local timer */
1424	x86_init.timers.setup_percpu_clockev();
 
 
 
 
 
 
 
 
 
1425
1426	smp_get_logical_apicid();
 
 
 
 
 
 
 
 
 
 
 
1427
1428	pr_info("CPU0: ");
 
 
 
 
 
 
 
 
1429	print_cpu_info(&cpu_data(0));
 
1430
1431	uv_system_init();
 
1432
1433	smp_quirk_init_udelay();
 
 
 
1434
1435	speculative_store_bypass_ht_init();
 
 
 
 
 
 
 
1436
1437	snp_set_wakeup_secondary_cpu();
 
 
1438}
1439
1440void arch_thaw_secondary_cpus_begin(void)
1441{
1442	set_cache_aps_delayed_init(true);
1443}
1444
1445void arch_thaw_secondary_cpus_end(void)
1446{
1447	cache_aps_init();
1448}
1449
1450/*
1451 * Early setup to make printk work.
1452 */
1453void __init native_smp_prepare_boot_cpu(void)
1454{
1455	int me = smp_processor_id();
1456
1457	/* SMP handles this from setup_per_cpu_areas() */
1458	if (!IS_ENABLED(CONFIG_SMP))
1459		switch_gdt_and_percpu_base(me);
1460
1461	/* already set me in cpu_online_mask in boot_cpu_init() */
1462	cpumask_set_cpu(me, cpu_callout_mask);
1463	cpu_set_state_online(me);
1464	native_pv_lock_init();
1465}
1466
1467void __init calculate_max_logical_packages(void)
1468{
1469	int ncpus;
1470
1471	/*
1472	 * Today neither Intel nor AMD support heterogeneous systems so
1473	 * extrapolate the boot cpu's data to all packages.
1474	 */
1475	ncpus = cpu_data(0).booted_cores * topology_max_smt_threads();
1476	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
1477	pr_info("Max logical packages: %u\n", __max_logical_packages);
1478}
1479
1480void __init native_smp_cpus_done(unsigned int max_cpus)
1481{
1482	pr_debug("Boot done\n");
1483
1484	calculate_max_logical_packages();
1485
1486	/* XXX for now assume numa-in-package and hybrid don't overlap */
1487	if (x86_has_numa_in_package)
1488		set_sched_topology(x86_numa_in_package_topology);
1489	if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU))
1490		set_sched_topology(x86_hybrid_topology);
1491
1492	nmi_selftest();
1493	impress_friends();
1494	cache_aps_init();
 
 
 
1495}
1496
1497static int __initdata setup_possible_cpus = -1;
1498static int __init _setup_possible_cpus(char *str)
1499{
1500	get_option(&str, &setup_possible_cpus);
1501	return 0;
1502}
1503early_param("possible_cpus", _setup_possible_cpus);
1504
1505
1506/*
1507 * cpu_possible_mask should be static, it cannot change as cpu's
1508 * are onlined, or offlined. The reason is per-cpu data-structures
1509 * are allocated by some modules at init time, and don't expect to
1510 * do this dynamically on cpu arrival/departure.
1511 * cpu_present_mask on the other hand can change dynamically.
1512 * In case when cpu_hotplug is not compiled, then we resort to current
1513 * behaviour, which is cpu_possible == cpu_present.
1514 * - Ashok Raj
1515 *
1516 * Three ways to find out the number of additional hotplug CPUs:
1517 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1518 * - The user can overwrite it with possible_cpus=NUM
1519 * - Otherwise don't reserve additional CPUs.
1520 * We do this because additional CPUs waste a lot of memory.
1521 * -AK
1522 */
1523__init void prefill_possible_map(void)
1524{
1525	int i, possible;
1526
1527	/* No boot processor was found in mptable or ACPI MADT */
1528	if (!num_processors) {
1529		if (boot_cpu_has(X86_FEATURE_APIC)) {
1530			int apicid = boot_cpu_physical_apicid;
1531			int cpu = hard_smp_processor_id();
1532
1533			pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu);
1534
1535			/* Make sure boot cpu is enumerated */
1536			if (apic->cpu_present_to_apicid(0) == BAD_APICID &&
1537			    apic->apic_id_valid(apicid))
1538				generic_processor_info(apicid, boot_cpu_apic_version);
1539		}
1540
1541		if (!num_processors)
1542			num_processors = 1;
1543	}
1544
1545	i = setup_max_cpus ?: 1;
1546	if (setup_possible_cpus == -1) {
1547		possible = num_processors;
1548#ifdef CONFIG_HOTPLUG_CPU
1549		if (setup_max_cpus)
1550			possible += disabled_cpus;
1551#else
1552		if (possible > i)
1553			possible = i;
1554#endif
1555	} else
1556		possible = setup_possible_cpus;
1557
1558	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
1559
1560	/* nr_cpu_ids could be reduced via nr_cpus= */
1561	if (possible > nr_cpu_ids) {
1562		pr_warn("%d Processors exceeds NR_CPUS limit of %u\n",
 
1563			possible, nr_cpu_ids);
1564		possible = nr_cpu_ids;
1565	}
1566
1567#ifdef CONFIG_HOTPLUG_CPU
1568	if (!setup_max_cpus)
1569#endif
1570	if (possible > i) {
1571		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
 
1572			possible, setup_max_cpus);
1573		possible = i;
1574	}
1575
1576	set_nr_cpu_ids(possible);
1577
1578	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1579		possible, max_t(int, possible - num_processors, 0));
1580
1581	reset_cpu_possible_mask();
1582
1583	for (i = 0; i < possible; i++)
1584		set_cpu_possible(i, true);
 
 
 
 
1585}
1586
1587#ifdef CONFIG_HOTPLUG_CPU
1588
1589/* Recompute SMT state for all CPUs on offline */
1590static void recompute_smt_state(void)
1591{
1592	int max_threads, cpu;
1593
1594	max_threads = 0;
1595	for_each_online_cpu (cpu) {
1596		int threads = cpumask_weight(topology_sibling_cpumask(cpu));
1597
1598		if (threads > max_threads)
1599			max_threads = threads;
1600	}
1601	__max_smt_threads = max_threads;
1602}
1603
1604static void remove_siblinginfo(int cpu)
1605{
1606	int sibling;
1607	struct cpuinfo_x86 *c = &cpu_data(cpu);
1608
1609	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
1610		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1611		/*/
1612		 * last thread sibling in this cpu core going down
1613		 */
1614		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1615			cpu_data(sibling).booted_cores--;
1616	}
1617
1618	for_each_cpu(sibling, topology_die_cpumask(cpu))
1619		cpumask_clear_cpu(cpu, topology_die_cpumask(sibling));
1620
1621	for_each_cpu(sibling, topology_sibling_cpumask(cpu)) {
1622		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1623		if (cpumask_weight(topology_sibling_cpumask(sibling)) == 1)
1624			cpu_data(sibling).smt_active = false;
1625	}
1626
1627	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
1628		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
1629	for_each_cpu(sibling, cpu_l2c_shared_mask(cpu))
1630		cpumask_clear_cpu(cpu, cpu_l2c_shared_mask(sibling));
1631	cpumask_clear(cpu_llc_shared_mask(cpu));
1632	cpumask_clear(cpu_l2c_shared_mask(cpu));
1633	cpumask_clear(topology_sibling_cpumask(cpu));
1634	cpumask_clear(topology_core_cpumask(cpu));
1635	cpumask_clear(topology_die_cpumask(cpu));
1636	c->cpu_core_id = 0;
1637	c->booted_cores = 0;
1638	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1639	recompute_smt_state();
1640}
1641
1642static void remove_cpu_from_maps(int cpu)
1643{
1644	set_cpu_online(cpu, false);
1645	cpumask_clear_cpu(cpu, cpu_callout_mask);
1646	cpumask_clear_cpu(cpu, cpu_callin_mask);
1647	/* was set by cpu_init() */
1648	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1649	numa_remove_cpu(cpu);
1650}
1651
1652void cpu_disable_common(void)
1653{
1654	int cpu = smp_processor_id();
1655
1656	remove_siblinginfo(cpu);
1657
1658	/* It's now safe to remove this processor from the online map */
1659	lock_vector_lock();
1660	remove_cpu_from_maps(cpu);
1661	unlock_vector_lock();
1662	fixup_irqs();
1663	lapic_offline();
1664}
1665
1666int native_cpu_disable(void)
1667{
1668	int ret;
1669
1670	ret = lapic_can_unplug_cpu();
1671	if (ret)
1672		return ret;
1673
1674	cpu_disable_common();
 
 
 
 
 
 
 
 
 
1675
1676        /*
1677         * Disable the local APIC. Otherwise IPI broadcasts will reach
1678         * it. It still responds normally to INIT, NMI, SMI, and SIPI
1679         * messages.
1680         *
1681         * Disabling the APIC must happen after cpu_disable_common()
1682         * which invokes fixup_irqs().
1683         *
1684         * Disabling the APIC preserves already set bits in IRR, but
1685         * an interrupt arriving after disabling the local APIC does not
1686         * set the corresponding IRR bit.
1687         *
1688         * fixup_irqs() scans IRR for set bits so it can raise a not
1689         * yet handled interrupt on the new destination CPU via an IPI
1690         * but obviously it can't do so for IRR bits which are not set.
1691         * IOW, interrupts arriving after disabling the local APIC will
1692         * be lost.
1693         */
1694	apic_soft_disable();
1695
 
1696	return 0;
1697}
1698
1699int common_cpu_die(unsigned int cpu)
1700{
1701	int ret = 0;
1702
1703	/* We don't do anything here: idle task is faking death itself. */
 
1704
1705	/* They ack this in play_dead() by setting CPU_DEAD */
1706	if (cpu_wait_death(cpu, 5)) {
1707		if (system_state == SYSTEM_RUNNING)
1708			pr_info("CPU %u is now offline\n", cpu);
1709	} else {
1710		pr_err("CPU %u didn't die...\n", cpu);
1711		ret = -1;
 
 
 
 
1712	}
1713
1714	return ret;
1715}
1716
1717void native_cpu_die(unsigned int cpu)
1718{
1719	common_cpu_die(cpu);
1720}
1721
1722void play_dead_common(void)
1723{
1724	idle_task_exit();
 
 
1725
 
1726	/* Ack it */
1727	(void)cpu_report_death();
1728
1729	/*
1730	 * With physical CPU hotplug, we should halt the cpu
1731	 */
1732	local_irq_disable();
1733}
1734
1735/**
1736 * cond_wakeup_cpu0 - Wake up CPU0 if needed.
1737 *
1738 * If NMI wants to wake up CPU0, start CPU0.
1739 */
1740void cond_wakeup_cpu0(void)
1741{
1742	if (smp_processor_id() == 0 && enable_start_cpu0)
1743		start_cpu0();
1744}
1745EXPORT_SYMBOL_GPL(cond_wakeup_cpu0);
1746
1747/*
1748 * We need to flush the caches before going to sleep, lest we have
1749 * dirty data in our caches when we come back up.
1750 */
1751static inline void mwait_play_dead(void)
1752{
1753	unsigned int eax, ebx, ecx, edx;
1754	unsigned int highest_cstate = 0;
1755	unsigned int highest_subcstate = 0;
1756	void *mwait_ptr;
1757	int i;
 
 
1758
1759	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1760	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
1761		return;
1762	if (!this_cpu_has(X86_FEATURE_MWAIT))
1763		return;
1764	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1765		return;
1766	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1767		return;
1768
1769	eax = CPUID_MWAIT_LEAF;
1770	ecx = 0;
1771	native_cpuid(&eax, &ebx, &ecx, &edx);
1772
1773	/*
1774	 * eax will be 0 if EDX enumeration is not valid.
1775	 * Initialized below to cstate, sub_cstate value when EDX is valid.
1776	 */
1777	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
1778		eax = 0;
1779	} else {
1780		edx >>= MWAIT_SUBSTATE_SIZE;
1781		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
1782			if (edx & MWAIT_SUBSTATE_MASK) {
1783				highest_cstate = i;
1784				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
1785			}
1786		}
1787		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
1788			(highest_subcstate - 1);
1789	}
1790
1791	/*
1792	 * This should be a memory location in a cache line which is
1793	 * unlikely to be touched by other processors.  The actual
1794	 * content is immaterial as it is not actually modified in any way.
1795	 */
1796	mwait_ptr = &current_thread_info()->flags;
1797
1798	wbinvd();
1799
1800	while (1) {
1801		/*
1802		 * The CLFLUSH is a workaround for erratum AAI65 for
1803		 * the Xeon 7400 series.  It's not clear it is actually
1804		 * needed, but it should be harmless in either case.
1805		 * The WBINVD is insufficient due to the spurious-wakeup
1806		 * case where we return around the loop.
1807		 */
1808		mb();
1809		clflush(mwait_ptr);
1810		mb();
1811		__monitor(mwait_ptr, 0, 0);
1812		mb();
1813		__mwait(eax, 0);
1814
1815		cond_wakeup_cpu0();
1816	}
1817}
1818
1819void hlt_play_dead(void)
1820{
1821	if (__this_cpu_read(cpu_info.x86) >= 4)
1822		wbinvd();
1823
1824	while (1) {
1825		native_halt();
1826
1827		cond_wakeup_cpu0();
1828	}
1829}
1830
1831void native_play_dead(void)
1832{
1833	play_dead_common();
1834	tboot_shutdown(TB_SHUTDOWN_WFS);
1835
1836	mwait_play_dead();	/* Only returns on failure */
1837	if (cpuidle_play_dead())
1838		hlt_play_dead();
1839}
1840
1841#else /* ... !CONFIG_HOTPLUG_CPU */
1842int native_cpu_disable(void)
1843{
1844	return -ENOSYS;
1845}
1846
1847void native_cpu_die(unsigned int cpu)
1848{
1849	/* We said "no" in __cpu_disable */
1850	BUG();
1851}
1852
1853void native_play_dead(void)
1854{
1855	BUG();
1856}
1857
1858#endif