1// SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3 *	x86 SMP booting functions
   4 *
   5 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
   6 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
   7 *	Copyright 2001 Andi Kleen, SuSE Labs.
   8 *
   9 *	Much of the core SMP work is based on previous work by Thomas Radke, to
  10 *	whom a great many thanks are extended.
  11 *
  12 *	Thanks to Intel for making available several different Pentium,
  13 *	Pentium Pro and Pentium-II/Xeon MP machines.
  14 *	Original development of Linux SMP code supported by Caldera.
  15 *
 
 
 
  16 *	Fixes
  17 *		Felix Koop	:	NR_CPUS used properly
  18 *		Jose Renau	:	Handle single CPU case.
  19 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
  20 *		Greg Wright	:	Fix for kernel stacks panic.
  21 *		Erich Boleyn	:	MP v1.4 and additional changes.
  22 *	Matthias Sattler	:	Changes for 2.1 kernel map.
  23 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
  24 *	Michael Chastain	:	Change trampoline.S to gnu as.
  25 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
  26 *		Ingo Molnar	:	Added APIC timers, based on code
  27 *					from Jose Renau
  28 *		Ingo Molnar	:	various cleanups and rewrites
  29 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
  30 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
  31 *	Andi Kleen		:	Changed for SMP boot into long mode.
  32 *		Martin J. Bligh	: 	Added support for multi-quad systems
  33 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
  34 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
  35 *      Andi Kleen              :       Converted to new state machine.
  36 *	Ashok Raj		: 	CPU hotplug support
  37 *	Glauber Costa		:	i386 and x86_64 integration
  38 */
  39
  40#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  41
  42#include <linux/init.h>
  43#include <linux/smp.h>
  44#include <linux/export.h>
  45#include <linux/sched.h>
  46#include <linux/sched/topology.h>
  47#include <linux/sched/hotplug.h>
  48#include <linux/sched/task_stack.h>
  49#include <linux/percpu.h>
  50#include <linux/memblock.h>
  51#include <linux/err.h>
  52#include <linux/nmi.h>
  53#include <linux/tboot.h>
 
  54#include <linux/gfp.h>
  55#include <linux/cpuidle.h>
  56#include <linux/numa.h>
  57#include <linux/pgtable.h>
  58#include <linux/overflow.h>
  59
  60#include <asm/acpi.h>
  61#include <asm/desc.h>
  62#include <asm/nmi.h>
  63#include <asm/irq.h>
  64#include <asm/realmode.h>
  65#include <asm/cpu.h>
  66#include <asm/numa.h>
 
  67#include <asm/tlbflush.h>
  68#include <asm/mtrr.h>
  69#include <asm/mwait.h>
  70#include <asm/apic.h>
  71#include <asm/io_apic.h>
  72#include <asm/fpu/internal.h>
  73#include <asm/setup.h>
  74#include <asm/uv/uv.h>
  75#include <linux/mc146818rtc.h>
  76#include <asm/i8259.h>
 
  77#include <asm/misc.h>
  78#include <asm/qspinlock.h>
  79#include <asm/intel-family.h>
  80#include <asm/cpu_device_id.h>
  81#include <asm/spec-ctrl.h>
  82#include <asm/hw_irq.h>
  83#include <asm/stackprotector.h>
 
  84
  85/* representing HT siblings of each logical CPU */
  86DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
  87EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  88
  89/* representing HT and core siblings of each logical CPU */
  90DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
  91EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  92
  93/* representing HT, core, and die siblings of each logical CPU */
  94DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map);
  95EXPORT_PER_CPU_SYMBOL(cpu_die_map);
  96
  97DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
  98
  99/* Per CPU bogomips and other parameters */
 100DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
 101EXPORT_PER_CPU_SYMBOL(cpu_info);
 102
 103/* Logical package management. We might want to allocate that dynamically */
 
 
 
 104unsigned int __max_logical_packages __read_mostly;
 105EXPORT_SYMBOL(__max_logical_packages);
 106static unsigned int logical_packages __read_mostly;
 107static unsigned int logical_die __read_mostly;
 108
 109/* Maximum number of SMT threads on any online core */
 110int __read_mostly __max_smt_threads = 1;
 111
 112/* Flag to indicate if a complete sched domain rebuild is required */
 113bool x86_topology_update;
 114
 115int arch_update_cpu_topology(void)
 116{
 117	int retval = x86_topology_update;
 118
 119	x86_topology_update = false;
 120	return retval;
 121}
 122
 123static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
 124{
 125	unsigned long flags;
 126
 127	spin_lock_irqsave(&rtc_lock, flags);
 128	CMOS_WRITE(0xa, 0xf);
 129	spin_unlock_irqrestore(&rtc_lock, flags);
 
 
 130	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
 131							start_eip >> 4;
 
 132	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
 133							start_eip & 0xf;
 
 134}
 135
 136static inline void smpboot_restore_warm_reset_vector(void)
 137{
 138	unsigned long flags;
 139
 140	/*
 
 
 
 
 
 141	 * Paranoid:  Set warm reset code and vector here back
 142	 * to default values.
 143	 */
 144	spin_lock_irqsave(&rtc_lock, flags);
 145	CMOS_WRITE(0, 0xf);
 146	spin_unlock_irqrestore(&rtc_lock, flags);
 147
 148	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
 149}
 150
 151static void init_freq_invariance(bool secondary);
 152
 153/*
 154 * Report back to the Boot Processor during boot time or to the caller processor
 155 * during CPU online.
 156 */
 157static void smp_callin(void)
 158{
 159	int cpuid;
 160
 161	/*
 162	 * If waken up by an INIT in an 82489DX configuration
 163	 * cpu_callout_mask guarantees we don't get here before
 164	 * an INIT_deassert IPI reaches our local APIC, so it is
 165	 * now safe to touch our local APIC.
 166	 */
 167	cpuid = smp_processor_id();
 168
 169	/*
 
 
 
 
 
 170	 * the boot CPU has finished the init stage and is spinning
 171	 * on callin_map until we finish. We are free to set up this
 172	 * CPU, first the APIC. (this is probably redundant on most
 173	 * boards)
 174	 */
 175	apic_ap_setup();
 176
 177	/*
 178	 * Save our processor parameters. Note: this information
 179	 * is needed for clock calibration.
 180	 */
 181	smp_store_cpu_info(cpuid);
 182
 183	/*
 184	 * The topology information must be up to date before
 185	 * calibrate_delay() and notify_cpu_starting().
 186	 */
 187	set_cpu_sibling_map(raw_smp_processor_id());
 188
 189	init_freq_invariance(true);
 190
 191	/*
 192	 * Get our bogomips.
 193	 * Update loops_per_jiffy in cpu_data. Previous call to
 194	 * smp_store_cpu_info() stored a value that is close but not as
 195	 * accurate as the value just calculated.
 196	 */
 197	calibrate_delay();
 198	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
 199	pr_debug("Stack at about %p\n", &cpuid);
 200
 
 
 
 
 
 201	wmb();
 202
 203	notify_cpu_starting(cpuid);
 204
 205	/*
 206	 * Allow the master to continue.
 207	 */
 208	cpumask_set_cpu(cpuid, cpu_callin_mask);
 209}
 210
 211static int cpu0_logical_apicid;
 212static int enable_start_cpu0;
 213/*
 214 * Activate a secondary processor.
 215 */
 216static void notrace start_secondary(void *unused)
 217{
 218	/*
 219	 * Don't put *anything* except direct CPU state initialization
 220	 * before cpu_init(), SMP booting is too fragile that we want to
 221	 * limit the things done here to the most necessary things.
 222	 */
 223	cr4_init();
 224
 225#ifdef CONFIG_X86_32
 226	/* switch away from the initial page table */
 227	load_cr3(swapper_pg_dir);
 228	__flush_tlb_all();
 229#endif
 230	load_current_idt();
 231	cpu_init();
 232	x86_cpuinit.early_percpu_clock_init();
 233	preempt_disable();
 234	smp_callin();
 235
 236	enable_start_cpu0 = 0;
 237
 
 
 
 
 
 
 238	/* otherwise gcc will move up smp_processor_id before the cpu_init */
 239	barrier();
 240	/*
 241	 * Check TSC synchronization with the boot CPU:
 242	 */
 243	check_tsc_sync_target();
 244
 245	speculative_store_bypass_ht_init();
 246
 247	/*
 248	 * Lock vector_lock, set CPU online and bring the vector
 249	 * allocator online. Online must be set with vector_lock held
 250	 * to prevent a concurrent irq setup/teardown from seeing a
 251	 * half valid vector space.
 252	 */
 253	lock_vector_lock();
 
 254	set_cpu_online(smp_processor_id(), true);
 255	lapic_online();
 256	unlock_vector_lock();
 257	cpu_set_state_online(smp_processor_id());
 258	x86_platform.nmi_init();
 259
 260	/* enable local interrupts */
 261	local_irq_enable();
 262
 
 
 
 263	x86_cpuinit.setup_percpu_clockev();
 264
 265	wmb();
 266	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 267}
 268
 269/**
 270 * topology_is_primary_thread - Check whether CPU is the primary SMT thread
 271 * @cpu:	CPU to check
 272 */
 273bool topology_is_primary_thread(unsigned int cpu)
 274{
 275	return apic_id_is_primary_thread(per_cpu(x86_cpu_to_apicid, cpu));
 276}
 277
 278/**
 279 * topology_smt_supported - Check whether SMT is supported by the CPUs
 280 */
 281bool topology_smt_supported(void)
 282{
 283	return smp_num_siblings > 1;
 284}
 285
 286/**
 287 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 288 *
 289 * Returns logical package id or -1 if not found
 290 */
 291int topology_phys_to_logical_pkg(unsigned int phys_pkg)
 292{
 293	int cpu;
 294
 295	for_each_possible_cpu(cpu) {
 296		struct cpuinfo_x86 *c = &cpu_data(cpu);
 297
 298		if (c->initialized && c->phys_proc_id == phys_pkg)
 299			return c->logical_proc_id;
 300	}
 301	return -1;
 302}
 303EXPORT_SYMBOL(topology_phys_to_logical_pkg);
 304/**
 305 * topology_phys_to_logical_die - Map a physical die id to logical
 306 *
 307 * Returns logical die id or -1 if not found
 308 */
 309int topology_phys_to_logical_die(unsigned int die_id, unsigned int cur_cpu)
 310{
 311	int cpu;
 312	int proc_id = cpu_data(cur_cpu).phys_proc_id;
 313
 314	for_each_possible_cpu(cpu) {
 315		struct cpuinfo_x86 *c = &cpu_data(cpu);
 316
 317		if (c->initialized && c->cpu_die_id == die_id &&
 318		    c->phys_proc_id == proc_id)
 319			return c->logical_die_id;
 320	}
 321	return -1;
 322}
 323EXPORT_SYMBOL(topology_phys_to_logical_die);
 324
 325/**
 326 * topology_update_package_map - Update the physical to logical package map
 327 * @pkg:	The physical package id as retrieved via CPUID
 328 * @cpu:	The cpu for which this is updated
 329 */
 330int topology_update_package_map(unsigned int pkg, unsigned int cpu)
 331{
 332	int new;
 
 
 
 
 
 
 
 333
 334	/* Already available somewhere? */
 335	new = topology_phys_to_logical_pkg(pkg);
 336	if (new >= 0)
 337		goto found;
 338
 
 
 
 
 
 
 339	new = logical_packages++;
 340	if (new != pkg) {
 341		pr_info("CPU %u Converting physical %u to logical package %u\n",
 342			cpu, pkg, new);
 343	}
 
 
 344found:
 345	cpu_data(cpu).logical_proc_id = new;
 346	return 0;
 347}
 
 348/**
 349 * topology_update_die_map - Update the physical to logical die map
 350 * @die:	The die id as retrieved via CPUID
 351 * @cpu:	The cpu for which this is updated
 352 */
 353int topology_update_die_map(unsigned int die, unsigned int cpu)
 354{
 355	int new;
 
 
 
 
 356
 357	/* Already available somewhere? */
 358	new = topology_phys_to_logical_die(die, cpu);
 359	if (new >= 0)
 360		goto found;
 361
 362	new = logical_die++;
 363	if (new != die) {
 364		pr_info("CPU %u Converting physical %u to logical die %u\n",
 365			cpu, die, new);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 366	}
 367found:
 368	cpu_data(cpu).logical_die_id = new;
 369	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 370}
 371
 372void __init smp_store_boot_cpu_info(void)
 373{
 374	int id = 0; /* CPU 0 */
 375	struct cpuinfo_x86 *c = &cpu_data(id);
 376
 377	*c = boot_cpu_data;
 378	c->cpu_index = id;
 379	topology_update_package_map(c->phys_proc_id, id);
 380	topology_update_die_map(c->cpu_die_id, id);
 381	c->initialized = true;
 382}
 383
 384/*
 385 * The bootstrap kernel entry code has set these up. Save them for
 386 * a given CPU
 387 */
 388void smp_store_cpu_info(int id)
 389{
 390	struct cpuinfo_x86 *c = &cpu_data(id);
 391
 392	/* Copy boot_cpu_data only on the first bringup */
 393	if (!c->initialized)
 394		*c = boot_cpu_data;
 395	c->cpu_index = id;
 396	/*
 397	 * During boot time, CPU0 has this setup already. Save the info when
 398	 * bringing up AP or offlined CPU0.
 399	 */
 400	identify_secondary_cpu(c);
 401	c->initialized = true;
 402}
 403
 404static bool
 405topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 406{
 407	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 408
 409	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
 410}
 411
 412static bool
 413topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
 414{
 415	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 416
 417	return !WARN_ONCE(!topology_same_node(c, o),
 418		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
 419		"[node: %d != %d]. Ignoring dependency.\n",
 420		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
 421}
 422
 423#define link_mask(mfunc, c1, c2)					\
 424do {									\
 425	cpumask_set_cpu((c1), mfunc(c2));				\
 426	cpumask_set_cpu((c2), mfunc(c1));				\
 427} while (0)
 428
 429static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 430{
 431	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
 432		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 433
 434		if (c->phys_proc_id == o->phys_proc_id &&
 435		    c->cpu_die_id == o->cpu_die_id &&
 436		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2)) {
 437			if (c->cpu_core_id == o->cpu_core_id)
 438				return topology_sane(c, o, "smt");
 439
 440			if ((c->cu_id != 0xff) &&
 441			    (o->cu_id != 0xff) &&
 442			    (c->cu_id == o->cu_id))
 443				return topology_sane(c, o, "smt");
 444		}
 445
 446	} else if (c->phys_proc_id == o->phys_proc_id &&
 447		   c->cpu_die_id == o->cpu_die_id &&
 448		   c->cpu_core_id == o->cpu_core_id) {
 449		return topology_sane(c, o, "smt");
 450	}
 451
 452	return false;
 453}
 454
 455/*
 456 * Define snc_cpu[] for SNC (Sub-NUMA Cluster) CPUs.
 457 *
 458 * These are Intel CPUs that enumerate an LLC that is shared by
 459 * multiple NUMA nodes. The LLC on these systems is shared for
 460 * off-package data access but private to the NUMA node (half
 461 * of the package) for on-package access.
 462 *
 463 * CPUID (the source of the information about the LLC) can only
 464 * enumerate the cache as being shared *or* unshared, but not
 465 * this particular configuration. The CPU in this case enumerates
 466 * the cache to be shared across the entire package (spanning both
 467 * NUMA nodes).
 468 */
 469
 470static const struct x86_cpu_id snc_cpu[] = {
 471	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
 472	{}
 473};
 474
 475static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 476{
 477	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;
 478
 479	/* Do not match if we do not have a valid APICID for cpu: */
 480	if (per_cpu(cpu_llc_id, cpu1) == BAD_APICID)
 481		return false;
 482
 483	/* Do not match if LLC id does not match: */
 484	if (per_cpu(cpu_llc_id, cpu1) != per_cpu(cpu_llc_id, cpu2))
 485		return false;
 486
 487	/*
 488	 * Allow the SNC topology without warning. Return of false
 489	 * means 'c' does not share the LLC of 'o'. This will be
 490	 * reflected to userspace.
 491	 */
 492	if (!topology_same_node(c, o) && x86_match_cpu(snc_cpu))
 493		return false;
 494
 495	return topology_sane(c, o, "llc");
 496}
 497
 498/*
 499 * Unlike the other levels, we do not enforce keeping a
 500 * multicore group inside a NUMA node.  If this happens, we will
 501 * discard the MC level of the topology later.
 502 */
 503static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 504{
 505	if (c->phys_proc_id == o->phys_proc_id)
 506		return true;
 507	return false;
 508}
 509
 510static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 511{
 512	if ((c->phys_proc_id == o->phys_proc_id) &&
 513		(c->cpu_die_id == o->cpu_die_id))
 514		return true;
 515	return false;
 516}
 517
 518
 519#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
 520static inline int x86_sched_itmt_flags(void)
 521{
 522	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;
 523}
 524
 525#ifdef CONFIG_SCHED_MC
 526static int x86_core_flags(void)
 527{
 528	return cpu_core_flags() | x86_sched_itmt_flags();
 529}
 530#endif
 531#ifdef CONFIG_SCHED_SMT
 532static int x86_smt_flags(void)
 533{
 534	return cpu_smt_flags() | x86_sched_itmt_flags();
 535}
 536#endif
 537#endif
 538
 539static struct sched_domain_topology_level x86_numa_in_package_topology[] = {
 540#ifdef CONFIG_SCHED_SMT
 541	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 542#endif
 543#ifdef CONFIG_SCHED_MC
 544	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 545#endif
 546	{ NULL, },
 547};
 548
 549static struct sched_domain_topology_level x86_topology[] = {
 550#ifdef CONFIG_SCHED_SMT
 551	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
 552#endif
 553#ifdef CONFIG_SCHED_MC
 554	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
 555#endif
 556	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
 557	{ NULL, },
 558};
 559
 560/*
 561 * Set if a package/die has multiple NUMA nodes inside.
 562 * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
 563 * Sub-NUMA Clustering have this.
 564 */
 565static bool x86_has_numa_in_package;
 566
 567void set_cpu_sibling_map(int cpu)
 568{
 569	bool has_smt = smp_num_siblings > 1;
 570	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
 571	struct cpuinfo_x86 *c = &cpu_data(cpu);
 572	struct cpuinfo_x86 *o;
 573	int i, threads;
 574
 575	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
 576
 577	if (!has_mp) {
 578		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
 579		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
 580		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
 581		cpumask_set_cpu(cpu, topology_die_cpumask(cpu));
 582		c->booted_cores = 1;
 583		return;
 584	}
 585
 586	for_each_cpu(i, cpu_sibling_setup_mask) {
 587		o = &cpu_data(i);
 588
 589		if ((i == cpu) || (has_smt && match_smt(c, o)))
 590			link_mask(topology_sibling_cpumask, cpu, i);
 591
 592		if ((i == cpu) || (has_mp && match_llc(c, o)))
 593			link_mask(cpu_llc_shared_mask, cpu, i);
 594
 595	}
 596
 597	/*
 598	 * This needs a separate iteration over the cpus because we rely on all
 599	 * topology_sibling_cpumask links to be set-up.
 600	 */
 601	for_each_cpu(i, cpu_sibling_setup_mask) {
 602		o = &cpu_data(i);
 603
 604		if ((i == cpu) || (has_mp && match_pkg(c, o))) {
 605			link_mask(topology_core_cpumask, cpu, i);
 606
 607			/*
 608			 *  Does this new cpu bringup a new core?
 609			 */
 610			if (cpumask_weight(
 611			    topology_sibling_cpumask(cpu)) == 1) {
 612				/*
 613				 * for each core in package, increment
 614				 * the booted_cores for this new cpu
 615				 */
 616				if (cpumask_first(
 617				    topology_sibling_cpumask(i)) == i)
 618					c->booted_cores++;
 619				/*
 620				 * increment the core count for all
 621				 * the other cpus in this package
 622				 */
 623				if (i != cpu)
 624					cpu_data(i).booted_cores++;
 625			} else if (i != cpu && !c->booted_cores)
 626				c->booted_cores = cpu_data(i).booted_cores;
 627		}
 628		if (match_pkg(c, o) && !topology_same_node(c, o))
 629			x86_has_numa_in_package = true;
 630
 631		if ((i == cpu) || (has_mp && match_die(c, o)))
 632			link_mask(topology_die_cpumask, cpu, i);
 633	}
 634
 635	threads = cpumask_weight(topology_sibling_cpumask(cpu));
 636	if (threads > __max_smt_threads)
 637		__max_smt_threads = threads;
 638}
 639
 640/* maps the cpu to the sched domain representing multi-core */
 641const struct cpumask *cpu_coregroup_mask(int cpu)
 642{
 643	return cpu_llc_shared_mask(cpu);
 644}
 645
 646static void impress_friends(void)
 647{
 648	int cpu;
 649	unsigned long bogosum = 0;
 650	/*
 651	 * Allow the user to impress friends.
 652	 */
 653	pr_debug("Before bogomips\n");
 654	for_each_possible_cpu(cpu)
 655		if (cpumask_test_cpu(cpu, cpu_callout_mask))
 656			bogosum += cpu_data(cpu).loops_per_jiffy;
 657	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
 658		num_online_cpus(),
 659		bogosum/(500000/HZ),
 660		(bogosum/(5000/HZ))%100);
 661
 662	pr_debug("Before bogocount - setting activated=1\n");
 663}
 664
 665void __inquire_remote_apic(int apicid)
 666{
 667	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
 668	const char * const names[] = { "ID", "VERSION", "SPIV" };
 669	int timeout;
 670	u32 status;
 671
 672	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
 673
 674	for (i = 0; i < ARRAY_SIZE(regs); i++) {
 675		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
 676
 677		/*
 678		 * Wait for idle.
 679		 */
 680		status = safe_apic_wait_icr_idle();
 681		if (status)
 682			pr_cont("a previous APIC delivery may have failed\n");
 683
 684		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
 685
 686		timeout = 0;
 687		do {
 688			udelay(100);
 689			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
 690		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
 691
 692		switch (status) {
 693		case APIC_ICR_RR_VALID:
 694			status = apic_read(APIC_RRR);
 695			pr_cont("%08x\n", status);
 696			break;
 697		default:
 698			pr_cont("failed\n");
 699		}
 700	}
 701}
 702
 703/*
 704 * The Multiprocessor Specification 1.4 (1997) example code suggests
 705 * that there should be a 10ms delay between the BSP asserting INIT
 706 * and de-asserting INIT, when starting a remote processor.
 707 * But that slows boot and resume on modern processors, which include
 708 * many cores and don't require that delay.
 709 *
 710 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
 711 * Modern processor families are quirked to remove the delay entirely.
 712 */
 713#define UDELAY_10MS_DEFAULT 10000
 714
 715static unsigned int init_udelay = UINT_MAX;
 716
 717static int __init cpu_init_udelay(char *str)
 718{
 719	get_option(&str, &init_udelay);
 720
 721	return 0;
 722}
 723early_param("cpu_init_udelay", cpu_init_udelay);
 724
 725static void __init smp_quirk_init_udelay(void)
 726{
 727	/* if cmdline changed it from default, leave it alone */
 728	if (init_udelay != UINT_MAX)
 729		return;
 730
 731	/* if modern processor, use no delay */
 732	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
 733	    ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && (boot_cpu_data.x86 >= 0x18)) ||
 734	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
 735		init_udelay = 0;
 736		return;
 737	}
 738	/* else, use legacy delay */
 739	init_udelay = UDELAY_10MS_DEFAULT;
 740}
 741
 742/*
 743 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 744 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 745 * won't ... remember to clear down the APIC, etc later.
 746 */
 747int
 748wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
 749{
 750	unsigned long send_status, accept_status = 0;
 751	int maxlvt;
 752
 753	/* Target chip */
 754	/* Boot on the stack */
 755	/* Kick the second */
 756	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
 757
 758	pr_debug("Waiting for send to finish...\n");
 759	send_status = safe_apic_wait_icr_idle();
 760
 761	/*
 762	 * Give the other CPU some time to accept the IPI.
 763	 */
 764	udelay(200);
 765	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 766		maxlvt = lapic_get_maxlvt();
 767		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
 768			apic_write(APIC_ESR, 0);
 769		accept_status = (apic_read(APIC_ESR) & 0xEF);
 770	}
 771	pr_debug("NMI sent\n");
 772
 773	if (send_status)
 774		pr_err("APIC never delivered???\n");
 775	if (accept_status)
 776		pr_err("APIC delivery error (%lx)\n", accept_status);
 777
 778	return (send_status | accept_status);
 779}
 780
 781static int
 782wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
 783{
 784	unsigned long send_status = 0, accept_status = 0;
 785	int maxlvt, num_starts, j;
 786
 787	maxlvt = lapic_get_maxlvt();
 788
 789	/*
 790	 * Be paranoid about clearing APIC errors.
 791	 */
 792	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
 793		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 794			apic_write(APIC_ESR, 0);
 795		apic_read(APIC_ESR);
 796	}
 797
 798	pr_debug("Asserting INIT\n");
 799
 800	/*
 801	 * Turn INIT on target chip
 802	 */
 803	/*
 804	 * Send IPI
 805	 */
 806	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
 807		       phys_apicid);
 808
 809	pr_debug("Waiting for send to finish...\n");
 810	send_status = safe_apic_wait_icr_idle();
 811
 812	udelay(init_udelay);
 813
 814	pr_debug("Deasserting INIT\n");
 815
 816	/* Target chip */
 817	/* Send IPI */
 818	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
 819
 820	pr_debug("Waiting for send to finish...\n");
 821	send_status = safe_apic_wait_icr_idle();
 822
 823	mb();
 824
 825	/*
 826	 * Should we send STARTUP IPIs ?
 827	 *
 828	 * Determine this based on the APIC version.
 829	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
 830	 */
 831	if (APIC_INTEGRATED(boot_cpu_apic_version))
 832		num_starts = 2;
 833	else
 834		num_starts = 0;
 835
 836	/*
 837	 * Run STARTUP IPI loop.
 838	 */
 839	pr_debug("#startup loops: %d\n", num_starts);
 840
 841	for (j = 1; j <= num_starts; j++) {
 842		pr_debug("Sending STARTUP #%d\n", j);
 843		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 844			apic_write(APIC_ESR, 0);
 845		apic_read(APIC_ESR);
 846		pr_debug("After apic_write\n");
 847
 848		/*
 849		 * STARTUP IPI
 850		 */
 851
 852		/* Target chip */
 853		/* Boot on the stack */
 854		/* Kick the second */
 855		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
 856			       phys_apicid);
 857
 858		/*
 859		 * Give the other CPU some time to accept the IPI.
 860		 */
 861		if (init_udelay == 0)
 862			udelay(10);
 863		else
 864			udelay(300);
 865
 866		pr_debug("Startup point 1\n");
 867
 868		pr_debug("Waiting for send to finish...\n");
 869		send_status = safe_apic_wait_icr_idle();
 870
 871		/*
 872		 * Give the other CPU some time to accept the IPI.
 873		 */
 874		if (init_udelay == 0)
 875			udelay(10);
 876		else
 877			udelay(200);
 878
 879		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
 880			apic_write(APIC_ESR, 0);
 881		accept_status = (apic_read(APIC_ESR) & 0xEF);
 882		if (send_status || accept_status)
 883			break;
 884	}
 885	pr_debug("After Startup\n");
 886
 887	if (send_status)
 888		pr_err("APIC never delivered???\n");
 889	if (accept_status)
 890		pr_err("APIC delivery error (%lx)\n", accept_status);
 891
 892	return (send_status | accept_status);
 893}
 894
 895/* reduce the number of lines printed when booting a large cpu count system */
 896static void announce_cpu(int cpu, int apicid)
 897{
 898	static int current_node = NUMA_NO_NODE;
 899	int node = early_cpu_to_node(cpu);
 900	static int width, node_width;
 901
 902	if (!width)
 903		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
 904
 905	if (!node_width)
 906		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */
 907
 908	if (cpu == 1)
 909		printk(KERN_INFO "x86: Booting SMP configuration:\n");
 910
 911	if (system_state < SYSTEM_RUNNING) {
 912		if (node != current_node) {
 913			if (current_node > (-1))
 914				pr_cont("\n");
 915			current_node = node;
 916
 917			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
 918			       node_width - num_digits(node), " ", node);
 919		}
 920
 921		/* Add padding for the BSP */
 922		if (cpu == 1)
 923			pr_cont("%*s", width + 1, " ");
 924
 925		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);
 926
 927	} else
 928		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
 929			node, cpu, apicid);
 930}
 931
 932static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
 933{
 934	int cpu;
 935
 936	cpu = smp_processor_id();
 937	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
 938		return NMI_HANDLED;
 939
 940	return NMI_DONE;
 941}
 942
 943/*
 944 * Wake up AP by INIT, INIT, STARTUP sequence.
 945 *
 946 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 947 * boot-strap code which is not a desired behavior for waking up BSP. To
 948 * void the boot-strap code, wake up CPU0 by NMI instead.
 949 *
 950 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 951 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 952 * We'll change this code in the future to wake up hard offlined CPU0 if
 953 * real platform and request are available.
 954 */
 955static int
 956wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
 957	       int *cpu0_nmi_registered)
 958{
 959	int id;
 960	int boot_error;
 961
 962	preempt_disable();
 963
 964	/*
 965	 * Wake up AP by INIT, INIT, STARTUP sequence.
 966	 */
 967	if (cpu) {
 968		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
 969		goto out;
 970	}
 971
 972	/*
 973	 * Wake up BSP by nmi.
 974	 *
 975	 * Register a NMI handler to help wake up CPU0.
 976	 */
 977	boot_error = register_nmi_handler(NMI_LOCAL,
 978					  wakeup_cpu0_nmi, 0, "wake_cpu0");
 979
 980	if (!boot_error) {
 981		enable_start_cpu0 = 1;
 982		*cpu0_nmi_registered = 1;
 983		if (apic->dest_logical == APIC_DEST_LOGICAL)
 984			id = cpu0_logical_apicid;
 985		else
 986			id = apicid;
 987		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
 988	}
 989
 990out:
 991	preempt_enable();
 992
 993	return boot_error;
 994}
 995
 996int common_cpu_up(unsigned int cpu, struct task_struct *idle)
 997{
 998	int ret;
 999
1000	/* Just in case we booted with a single CPU. */
1001	alternatives_enable_smp();
1002
1003	per_cpu(current_task, cpu) = idle;
1004	cpu_init_stack_canary(cpu, idle);
1005
1006	/* Initialize the interrupt stack(s) */
1007	ret = irq_init_percpu_irqstack(cpu);
1008	if (ret)
1009		return ret;
1010
1011#ifdef CONFIG_X86_32
1012	/* Stack for startup_32 can be just as for start_secondary onwards */
1013	per_cpu(cpu_current_top_of_stack, cpu) = task_top_of_stack(idle);
 
 
1014#else
1015	initial_gs = per_cpu_offset(cpu);
1016#endif
1017	return 0;
1018}
1019
1020/*
1021 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
1022 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
1023 * Returns zero if CPU booted OK, else error code from
1024 * ->wakeup_secondary_cpu.
1025 */
1026static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
1027		       int *cpu0_nmi_registered)
1028{
 
 
1029	/* start_ip had better be page-aligned! */
1030	unsigned long start_ip = real_mode_header->trampoline_start;
1031
1032	unsigned long boot_error = 0;
 
1033	unsigned long timeout;
1034
1035	idle->thread.sp = (unsigned long)task_pt_regs(idle);
1036	early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
1037	initial_code = (unsigned long)start_secondary;
1038	initial_stack  = idle->thread.sp;
1039
1040	/* Enable the espfix hack for this CPU */
 
 
 
1041	init_espfix_ap(cpu);
 
1042
1043	/* So we see what's up */
1044	announce_cpu(cpu, apicid);
1045
1046	/*
1047	 * This grunge runs the startup process for
1048	 * the targeted processor.
1049	 */
1050
1051	if (x86_platform.legacy.warm_reset) {
1052
1053		pr_debug("Setting warm reset code and vector.\n");
1054
1055		smpboot_setup_warm_reset_vector(start_ip);
1056		/*
1057		 * Be paranoid about clearing APIC errors.
1058		*/
1059		if (APIC_INTEGRATED(boot_cpu_apic_version)) {
1060			apic_write(APIC_ESR, 0);
1061			apic_read(APIC_ESR);
1062		}
1063	}
1064
1065	/*
1066	 * AP might wait on cpu_callout_mask in cpu_init() with
1067	 * cpu_initialized_mask set if previous attempt to online
1068	 * it timed-out. Clear cpu_initialized_mask so that after
1069	 * INIT/SIPI it could start with a clean state.
1070	 */
1071	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1072	smp_mb();
1073
1074	/*
1075	 * Wake up a CPU in difference cases:
1076	 * - Use the method in the APIC driver if it's defined
1077	 * Otherwise,
1078	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1079	 */
1080	if (apic->wakeup_secondary_cpu)
1081		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
1082	else
1083		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
1084						     cpu0_nmi_registered);
1085
1086	if (!boot_error) {
1087		/*
1088		 * Wait 10s total for first sign of life from AP
1089		 */
1090		boot_error = -1;
1091		timeout = jiffies + 10*HZ;
1092		while (time_before(jiffies, timeout)) {
1093			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
1094				/*
1095				 * Tell AP to proceed with initialization
1096				 */
1097				cpumask_set_cpu(cpu, cpu_callout_mask);
1098				boot_error = 0;
1099				break;
1100			}
1101			schedule();
1102		}
1103	}
1104
1105	if (!boot_error) {
1106		/*
1107		 * Wait till AP completes initial initialization
1108		 */
1109		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
1110			/*
1111			 * Allow other tasks to run while we wait for the
1112			 * AP to come online. This also gives a chance
1113			 * for the MTRR work(triggered by the AP coming online)
1114			 * to be completed in the stop machine context.
1115			 */
1116			schedule();
1117		}
1118	}
1119
1120	if (x86_platform.legacy.warm_reset) {
 
 
 
1121		/*
1122		 * Cleanup possible dangling ends...
1123		 */
1124		smpboot_restore_warm_reset_vector();
1125	}
 
 
 
 
 
 
1126
1127	return boot_error;
1128}
1129
1130int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1131{
1132	int apicid = apic->cpu_present_to_apicid(cpu);
1133	int cpu0_nmi_registered = 0;
1134	unsigned long flags;
1135	int err, ret = 0;
1136
1137	lockdep_assert_irqs_enabled();
1138
1139	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1140
1141	if (apicid == BAD_APICID ||
1142	    !physid_isset(apicid, phys_cpu_present_map) ||
1143	    !apic->apic_id_valid(apicid)) {
1144		pr_err("%s: bad cpu %d\n", __func__, cpu);
1145		return -EINVAL;
1146	}
1147
1148	/*
1149	 * Already booted CPU?
1150	 */
1151	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1152		pr_debug("do_boot_cpu %d Already started\n", cpu);
1153		return -ENOSYS;
1154	}
1155
1156	/*
1157	 * Save current MTRR state in case it was changed since early boot
1158	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
1159	 */
1160	mtrr_save_state();
1161
1162	/* x86 CPUs take themselves offline, so delayed offline is OK. */
1163	err = cpu_check_up_prepare(cpu);
1164	if (err && err != -EBUSY)
1165		return err;
1166
1167	/* the FPU context is blank, nobody can own it */
1168	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
1169
1170	err = common_cpu_up(cpu, tidle);
1171	if (err)
1172		return err;
1173
1174	err = do_boot_cpu(apicid, cpu, tidle, &cpu0_nmi_registered);
1175	if (err) {
1176		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1177		ret = -EIO;
1178		goto unreg_nmi;
1179	}
1180
1181	/*
1182	 * Check TSC synchronization with the AP (keep irqs disabled
1183	 * while doing so):
1184	 */
1185	local_irq_save(flags);
1186	check_tsc_sync_source(cpu);
1187	local_irq_restore(flags);
1188
1189	while (!cpu_online(cpu)) {
1190		cpu_relax();
1191		touch_nmi_watchdog();
1192	}
1193
1194unreg_nmi:
1195	/*
1196	 * Clean up the nmi handler. Do this after the callin and callout sync
1197	 * to avoid impact of possible long unregister time.
1198	 */
1199	if (cpu0_nmi_registered)
1200		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");
1201
1202	return ret;
1203}
1204
1205/**
1206 * arch_disable_smp_support() - disables SMP support for x86 at runtime
1207 */
1208void arch_disable_smp_support(void)
1209{
1210	disable_ioapic_support();
1211}
1212
1213/*
1214 * Fall back to non SMP mode after errors.
1215 *
1216 * RED-PEN audit/test this more. I bet there is more state messed up here.
1217 */
1218static __init void disable_smp(void)
1219{
1220	pr_info("SMP disabled\n");
1221
1222	disable_ioapic_support();
1223
1224	init_cpu_present(cpumask_of(0));
1225	init_cpu_possible(cpumask_of(0));
1226
1227	if (smp_found_config)
1228		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1229	else
1230		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1231	cpumask_set_cpu(0, topology_sibling_cpumask(0));
1232	cpumask_set_cpu(0, topology_core_cpumask(0));
1233	cpumask_set_cpu(0, topology_die_cpumask(0));
1234}
1235
 
 
 
 
 
 
 
1236/*
1237 * Various sanity checks.
1238 */
1239static void __init smp_sanity_check(void)
1240{
1241	preempt_disable();
1242
1243#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1244	if (def_to_bigsmp && nr_cpu_ids > 8) {
1245		unsigned int cpu;
1246		unsigned nr;
1247
1248		pr_warn("More than 8 CPUs detected - skipping them\n"
1249			"Use CONFIG_X86_BIGSMP\n");
1250
1251		nr = 0;
1252		for_each_present_cpu(cpu) {
1253			if (nr >= 8)
1254				set_cpu_present(cpu, false);
1255			nr++;
1256		}
1257
1258		nr = 0;
1259		for_each_possible_cpu(cpu) {
1260			if (nr >= 8)
1261				set_cpu_possible(cpu, false);
1262			nr++;
1263		}
1264
1265		nr_cpu_ids = 8;
1266	}
1267#endif
1268
1269	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1270		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
1271			hard_smp_processor_id());
1272
1273		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1274	}
1275
1276	/*
 
 
 
 
 
 
 
 
 
 
1277	 * Should not be necessary because the MP table should list the boot
1278	 * CPU too, but we do it for the sake of robustness anyway.
1279	 */
1280	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1281		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
1282			  boot_cpu_physical_apicid);
1283		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
1284	}
1285	preempt_enable();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1286}
1287
1288static void __init smp_cpu_index_default(void)
1289{
1290	int i;
1291	struct cpuinfo_x86 *c;
1292
1293	for_each_possible_cpu(i) {
1294		c = &cpu_data(i);
1295		/* mark all to hotplug */
1296		c->cpu_index = nr_cpu_ids;
1297	}
1298}
1299
1300static void __init smp_get_logical_apicid(void)
1301{
1302	if (x2apic_mode)
1303		cpu0_logical_apicid = apic_read(APIC_LDR);
1304	else
1305		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
1306}
1307
1308/*
1309 * Prepare for SMP bootup.
1310 * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1311 *            for common interface support.
1312 */
1313void __init native_smp_prepare_cpus(unsigned int max_cpus)
1314{
1315	unsigned int i;
1316
1317	smp_cpu_index_default();
1318
1319	/*
1320	 * Setup boot CPU information
1321	 */
1322	smp_store_boot_cpu_info(); /* Final full version of the data */
1323	cpumask_copy(cpu_callin_mask, cpumask_of(0));
1324	mb();
1325
1326	for_each_possible_cpu(i) {
1327		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
1328		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1329		zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
1330		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1331	}
1332
1333	/*
1334	 * Set 'default' x86 topology, this matches default_topology() in that
1335	 * it has NUMA nodes as a topology level. See also
1336	 * native_smp_cpus_done().
1337	 *
1338	 * Must be done before set_cpus_sibling_map() is ran.
1339	 */
1340	set_sched_topology(x86_topology);
1341
1342	set_cpu_sibling_map(0);
1343	init_freq_invariance(false);
1344	smp_sanity_check();
1345
1346	switch (apic_intr_mode) {
1347	case APIC_PIC:
1348	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1349		disable_smp();
 
 
1350		return;
1351	case APIC_SYMMETRIC_IO_NO_ROUTING:
1352		disable_smp();
1353		/* Setup local timer */
1354		x86_init.timers.setup_percpu_clockev();
1355		return;
1356	case APIC_VIRTUAL_WIRE:
1357	case APIC_SYMMETRIC_IO:
 
 
 
1358		break;
1359	}
1360
1361	/* Setup local timer */
1362	x86_init.timers.setup_percpu_clockev();
 
 
 
1363
1364	smp_get_logical_apicid();
 
1365
1366	pr_info("CPU0: ");
1367	print_cpu_info(&cpu_data(0));
1368
1369	uv_system_init();
 
1370
1371	set_mtrr_aps_delayed_init();
1372
1373	smp_quirk_init_udelay();
1374
1375	speculative_store_bypass_ht_init();
1376}
1377
1378void arch_thaw_secondary_cpus_begin(void)
1379{
1380	set_mtrr_aps_delayed_init();
1381}
1382
1383void arch_thaw_secondary_cpus_end(void)
1384{
1385	mtrr_aps_init();
1386}
1387
1388/*
1389 * Early setup to make printk work.
1390 */
1391void __init native_smp_prepare_boot_cpu(void)
1392{
1393	int me = smp_processor_id();
1394	switch_to_new_gdt(me);
1395	/* already set me in cpu_online_mask in boot_cpu_init() */
1396	cpumask_set_cpu(me, cpu_callout_mask);
1397	cpu_set_state_online(me);
1398	native_pv_lock_init();
1399}
1400
1401void __init calculate_max_logical_packages(void)
1402{
1403	int ncpus;
1404
1405	/*
1406	 * Today neither Intel nor AMD support heterogenous systems so
1407	 * extrapolate the boot cpu's data to all packages.
1408	 */
1409	ncpus = cpu_data(0).booted_cores * topology_max_smt_threads();
1410	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
1411	pr_info("Max logical packages: %u\n", __max_logical_packages);
1412}
1413
1414void __init native_smp_cpus_done(unsigned int max_cpus)
1415{
1416	pr_debug("Boot done\n");
1417
1418	calculate_max_logical_packages();
1419
1420	if (x86_has_numa_in_package)
1421		set_sched_topology(x86_numa_in_package_topology);
1422
1423	nmi_selftest();
1424	impress_friends();
 
1425	mtrr_aps_init();
1426}
1427
1428static int __initdata setup_possible_cpus = -1;
1429static int __init _setup_possible_cpus(char *str)
1430{
1431	get_option(&str, &setup_possible_cpus);
1432	return 0;
1433}
1434early_param("possible_cpus", _setup_possible_cpus);
1435
1436
1437/*
1438 * cpu_possible_mask should be static, it cannot change as cpu's
1439 * are onlined, or offlined. The reason is per-cpu data-structures
1440 * are allocated by some modules at init time, and don't expect to
1441 * do this dynamically on cpu arrival/departure.
1442 * cpu_present_mask on the other hand can change dynamically.
1443 * In case when cpu_hotplug is not compiled, then we resort to current
1444 * behaviour, which is cpu_possible == cpu_present.
1445 * - Ashok Raj
1446 *
1447 * Three ways to find out the number of additional hotplug CPUs:
1448 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1449 * - The user can overwrite it with possible_cpus=NUM
1450 * - Otherwise don't reserve additional CPUs.
1451 * We do this because additional CPUs waste a lot of memory.
1452 * -AK
1453 */
1454__init void prefill_possible_map(void)
1455{
1456	int i, possible;
1457
1458	/* No boot processor was found in mptable or ACPI MADT */
1459	if (!num_processors) {
1460		if (boot_cpu_has(X86_FEATURE_APIC)) {
1461			int apicid = boot_cpu_physical_apicid;
1462			int cpu = hard_smp_processor_id();
1463
1464			pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu);
1465
1466			/* Make sure boot cpu is enumerated */
1467			if (apic->cpu_present_to_apicid(0) == BAD_APICID &&
1468			    apic->apic_id_valid(apicid))
1469				generic_processor_info(apicid, boot_cpu_apic_version);
1470		}
1471
1472		if (!num_processors)
1473			num_processors = 1;
1474	}
1475
1476	i = setup_max_cpus ?: 1;
1477	if (setup_possible_cpus == -1) {
1478		possible = num_processors;
1479#ifdef CONFIG_HOTPLUG_CPU
1480		if (setup_max_cpus)
1481			possible += disabled_cpus;
1482#else
1483		if (possible > i)
1484			possible = i;
1485#endif
1486	} else
1487		possible = setup_possible_cpus;
1488
1489	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
1490
1491	/* nr_cpu_ids could be reduced via nr_cpus= */
1492	if (possible > nr_cpu_ids) {
1493		pr_warn("%d Processors exceeds NR_CPUS limit of %u\n",
1494			possible, nr_cpu_ids);
1495		possible = nr_cpu_ids;
1496	}
1497
1498#ifdef CONFIG_HOTPLUG_CPU
1499	if (!setup_max_cpus)
1500#endif
1501	if (possible > i) {
1502		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1503			possible, setup_max_cpus);
1504		possible = i;
1505	}
1506
1507	nr_cpu_ids = possible;
1508
1509	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1510		possible, max_t(int, possible - num_processors, 0));
1511
1512	reset_cpu_possible_mask();
1513
1514	for (i = 0; i < possible; i++)
1515		set_cpu_possible(i, true);
1516}
1517
1518#ifdef CONFIG_HOTPLUG_CPU
1519
1520/* Recompute SMT state for all CPUs on offline */
1521static void recompute_smt_state(void)
1522{
1523	int max_threads, cpu;
1524
1525	max_threads = 0;
1526	for_each_online_cpu (cpu) {
1527		int threads = cpumask_weight(topology_sibling_cpumask(cpu));
1528
1529		if (threads > max_threads)
1530			max_threads = threads;
1531	}
1532	__max_smt_threads = max_threads;
1533}
1534
1535static void remove_siblinginfo(int cpu)
1536{
1537	int sibling;
1538	struct cpuinfo_x86 *c = &cpu_data(cpu);
1539
1540	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
1541		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1542		/*/
1543		 * last thread sibling in this cpu core going down
1544		 */
1545		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1546			cpu_data(sibling).booted_cores--;
1547	}
1548
1549	for_each_cpu(sibling, topology_die_cpumask(cpu))
1550		cpumask_clear_cpu(cpu, topology_die_cpumask(sibling));
1551	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
1552		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1553	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
1554		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
1555	cpumask_clear(cpu_llc_shared_mask(cpu));
1556	cpumask_clear(topology_sibling_cpumask(cpu));
1557	cpumask_clear(topology_core_cpumask(cpu));
1558	cpumask_clear(topology_die_cpumask(cpu));
1559	c->cpu_core_id = 0;
1560	c->booted_cores = 0;
1561	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1562	recompute_smt_state();
1563}
1564
1565static void remove_cpu_from_maps(int cpu)
1566{
1567	set_cpu_online(cpu, false);
1568	cpumask_clear_cpu(cpu, cpu_callout_mask);
1569	cpumask_clear_cpu(cpu, cpu_callin_mask);
1570	/* was set by cpu_init() */
1571	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1572	numa_remove_cpu(cpu);
1573}
1574
1575void cpu_disable_common(void)
1576{
1577	int cpu = smp_processor_id();
1578
1579	remove_siblinginfo(cpu);
1580
1581	/* It's now safe to remove this processor from the online map */
1582	lock_vector_lock();
1583	remove_cpu_from_maps(cpu);
1584	unlock_vector_lock();
1585	fixup_irqs();
1586	lapic_offline();
1587}
1588
1589int native_cpu_disable(void)
1590{
1591	int ret;
1592
1593	ret = lapic_can_unplug_cpu();
1594	if (ret)
1595		return ret;
1596
 
1597	cpu_disable_common();
1598
1599        /*
1600         * Disable the local APIC. Otherwise IPI broadcasts will reach
1601         * it. It still responds normally to INIT, NMI, SMI, and SIPI
1602         * messages.
1603         *
1604         * Disabling the APIC must happen after cpu_disable_common()
1605         * which invokes fixup_irqs().
1606         *
1607         * Disabling the APIC preserves already set bits in IRR, but
1608         * an interrupt arriving after disabling the local APIC does not
1609         * set the corresponding IRR bit.
1610         *
1611         * fixup_irqs() scans IRR for set bits so it can raise a not
1612         * yet handled interrupt on the new destination CPU via an IPI
1613         * but obviously it can't do so for IRR bits which are not set.
1614         * IOW, interrupts arriving after disabling the local APIC will
1615         * be lost.
1616         */
1617	apic_soft_disable();
1618
1619	return 0;
1620}
1621
1622int common_cpu_die(unsigned int cpu)
1623{
1624	int ret = 0;
1625
1626	/* We don't do anything here: idle task is faking death itself. */
1627
1628	/* They ack this in play_dead() by setting CPU_DEAD */
1629	if (cpu_wait_death(cpu, 5)) {
1630		if (system_state == SYSTEM_RUNNING)
1631			pr_info("CPU %u is now offline\n", cpu);
1632	} else {
1633		pr_err("CPU %u didn't die...\n", cpu);
1634		ret = -1;
1635	}
1636
1637	return ret;
1638}
1639
1640void native_cpu_die(unsigned int cpu)
1641{
1642	common_cpu_die(cpu);
1643}
1644
1645void play_dead_common(void)
1646{
1647	idle_task_exit();
 
1648
1649	/* Ack it */
1650	(void)cpu_report_death();
1651
1652	/*
1653	 * With physical CPU hotplug, we should halt the cpu
1654	 */
1655	local_irq_disable();
1656}
1657
1658static bool wakeup_cpu0(void)
1659{
1660	if (smp_processor_id() == 0 && enable_start_cpu0)
1661		return true;
1662
1663	return false;
1664}
1665
1666/*
1667 * We need to flush the caches before going to sleep, lest we have
1668 * dirty data in our caches when we come back up.
1669 */
1670static inline void mwait_play_dead(void)
1671{
1672	unsigned int eax, ebx, ecx, edx;
1673	unsigned int highest_cstate = 0;
1674	unsigned int highest_subcstate = 0;
1675	void *mwait_ptr;
1676	int i;
1677
1678	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1679	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
1680		return;
1681	if (!this_cpu_has(X86_FEATURE_MWAIT))
1682		return;
1683	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1684		return;
1685	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1686		return;
1687
1688	eax = CPUID_MWAIT_LEAF;
1689	ecx = 0;
1690	native_cpuid(&eax, &ebx, &ecx, &edx);
1691
1692	/*
1693	 * eax will be 0 if EDX enumeration is not valid.
1694	 * Initialized below to cstate, sub_cstate value when EDX is valid.
1695	 */
1696	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
1697		eax = 0;
1698	} else {
1699		edx >>= MWAIT_SUBSTATE_SIZE;
1700		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
1701			if (edx & MWAIT_SUBSTATE_MASK) {
1702				highest_cstate = i;
1703				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
1704			}
1705		}
1706		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
1707			(highest_subcstate - 1);
1708	}
1709
1710	/*
1711	 * This should be a memory location in a cache line which is
1712	 * unlikely to be touched by other processors.  The actual
1713	 * content is immaterial as it is not actually modified in any way.
1714	 */
1715	mwait_ptr = &current_thread_info()->flags;
1716
1717	wbinvd();
1718
1719	while (1) {
1720		/*
1721		 * The CLFLUSH is a workaround for erratum AAI65 for
1722		 * the Xeon 7400 series.  It's not clear it is actually
1723		 * needed, but it should be harmless in either case.
1724		 * The WBINVD is insufficient due to the spurious-wakeup
1725		 * case where we return around the loop.
1726		 */
1727		mb();
1728		clflush(mwait_ptr);
1729		mb();
1730		__monitor(mwait_ptr, 0, 0);
1731		mb();
1732		__mwait(eax, 0);
1733		/*
1734		 * If NMI wants to wake up CPU0, start CPU0.
1735		 */
1736		if (wakeup_cpu0())
1737			start_cpu0();
1738	}
1739}
1740
1741void hlt_play_dead(void)
1742{
1743	if (__this_cpu_read(cpu_info.x86) >= 4)
1744		wbinvd();
1745
1746	while (1) {
1747		native_halt();
1748		/*
1749		 * If NMI wants to wake up CPU0, start CPU0.
1750		 */
1751		if (wakeup_cpu0())
1752			start_cpu0();
1753	}
1754}
1755
1756void native_play_dead(void)
1757{
1758	play_dead_common();
1759	tboot_shutdown(TB_SHUTDOWN_WFS);
1760
1761	mwait_play_dead();	/* Only returns on failure */
1762	if (cpuidle_play_dead())
1763		hlt_play_dead();
1764}
1765
1766#else /* ... !CONFIG_HOTPLUG_CPU */
1767int native_cpu_disable(void)
1768{
1769	return -ENOSYS;
1770}
1771
1772void native_cpu_die(unsigned int cpu)
1773{
1774	/* We said "no" in __cpu_disable */
1775	BUG();
1776}
1777
1778void native_play_dead(void)
1779{
1780	BUG();
1781}
1782
1783#endif
1784
1785#ifdef CONFIG_X86_64
1786/*
1787 * APERF/MPERF frequency ratio computation.
1788 *
1789 * The scheduler wants to do frequency invariant accounting and needs a <1
1790 * ratio to account for the 'current' frequency, corresponding to
1791 * freq_curr / freq_max.
1792 *
1793 * Since the frequency freq_curr on x86 is controlled by micro-controller and
1794 * our P-state setting is little more than a request/hint, we need to observe
1795 * the effective frequency 'BusyMHz', i.e. the average frequency over a time
1796 * interval after discarding idle time. This is given by:
1797 *
1798 *   BusyMHz = delta_APERF / delta_MPERF * freq_base
1799 *
1800 * where freq_base is the max non-turbo P-state.
1801 *
1802 * The freq_max term has to be set to a somewhat arbitrary value, because we
1803 * can't know which turbo states will be available at a given point in time:
1804 * it all depends on the thermal headroom of the entire package. We set it to
1805 * the turbo level with 4 cores active.
1806 *
1807 * Benchmarks show that's a good compromise between the 1C turbo ratio
1808 * (freq_curr/freq_max would rarely reach 1) and something close to freq_base,
1809 * which would ignore the entire turbo range (a conspicuous part, making
1810 * freq_curr/freq_max always maxed out).
1811 *
1812 * An exception to the heuristic above is the Atom uarch, where we choose the
1813 * highest turbo level for freq_max since Atom's are generally oriented towards
1814 * power efficiency.
1815 *
1816 * Setting freq_max to anything less than the 1C turbo ratio makes the ratio
1817 * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
1818 */
1819
1820DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);
1821
1822static DEFINE_PER_CPU(u64, arch_prev_aperf);
1823static DEFINE_PER_CPU(u64, arch_prev_mperf);
1824static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE;
1825static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE;
1826
1827void arch_set_max_freq_ratio(bool turbo_disabled)
1828{
1829	arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE :
1830					arch_turbo_freq_ratio;
1831}
1832
1833static bool turbo_disabled(void)
1834{
1835	u64 misc_en;
1836	int err;
1837
1838	err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en);
1839	if (err)
1840		return false;
1841
1842	return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
1843}
1844
1845static bool slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
1846{
1847	int err;
1848
1849	err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq);
1850	if (err)
1851		return false;
1852
1853	err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq);
1854	if (err)
1855		return false;
1856
1857	*base_freq = (*base_freq >> 16) & 0x3F;     /* max P state */
1858	*turbo_freq = *turbo_freq & 0x3F;           /* 1C turbo    */
1859
1860	return true;
1861}
1862
1863#include <asm/cpu_device_id.h>
1864#include <asm/intel-family.h>
1865
1866#define X86_MATCH(model)					\
1867	X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6,		\
1868		INTEL_FAM6_##model, X86_FEATURE_APERFMPERF, NULL)
1869
1870static const struct x86_cpu_id has_knl_turbo_ratio_limits[] = {
1871	X86_MATCH(XEON_PHI_KNL),
1872	X86_MATCH(XEON_PHI_KNM),
1873	{}
1874};
1875
1876static const struct x86_cpu_id has_skx_turbo_ratio_limits[] = {
1877	X86_MATCH(SKYLAKE_X),
1878	{}
1879};
1880
1881static const struct x86_cpu_id has_glm_turbo_ratio_limits[] = {
1882	X86_MATCH(ATOM_GOLDMONT),
1883	X86_MATCH(ATOM_GOLDMONT_D),
1884	X86_MATCH(ATOM_GOLDMONT_PLUS),
1885	{}
1886};
1887
1888static bool knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq,
1889				int num_delta_fratio)
1890{
1891	int fratio, delta_fratio, found;
1892	int err, i;
1893	u64 msr;
1894
1895	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
1896	if (err)
1897		return false;
1898
1899	*base_freq = (*base_freq >> 8) & 0xFF;	    /* max P state */
1900
1901	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
1902	if (err)
1903		return false;
1904
1905	fratio = (msr >> 8) & 0xFF;
1906	i = 16;
1907	found = 0;
1908	do {
1909		if (found >= num_delta_fratio) {
1910			*turbo_freq = fratio;
1911			return true;
1912		}
1913
1914		delta_fratio = (msr >> (i + 5)) & 0x7;
1915
1916		if (delta_fratio) {
1917			found += 1;
1918			fratio -= delta_fratio;
1919		}
1920
1921		i += 8;
1922	} while (i < 64);
1923
1924	return true;
1925}
1926
1927static bool skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size)
1928{
1929	u64 ratios, counts;
1930	u32 group_size;
1931	int err, i;
1932
1933	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
1934	if (err)
1935		return false;
1936
1937	*base_freq = (*base_freq >> 8) & 0xFF;      /* max P state */
1938
1939	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios);
1940	if (err)
1941		return false;
1942
1943	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts);
1944	if (err)
1945		return false;
1946
1947	for (i = 0; i < 64; i += 8) {
1948		group_size = (counts >> i) & 0xFF;
1949		if (group_size >= size) {
1950			*turbo_freq = (ratios >> i) & 0xFF;
1951			return true;
1952		}
1953	}
1954
1955	return false;
1956}
1957
1958static bool core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
1959{
1960	u64 msr;
1961	int err;
1962
1963	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
1964	if (err)
1965		return false;
1966
1967	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
1968	if (err)
1969		return false;
1970
1971	*base_freq = (*base_freq >> 8) & 0xFF;    /* max P state */
1972	*turbo_freq = (msr >> 24) & 0xFF;         /* 4C turbo    */
1973
1974	/* The CPU may have less than 4 cores */
1975	if (!*turbo_freq)
1976		*turbo_freq = msr & 0xFF;         /* 1C turbo    */
1977
1978	return true;
1979}
1980
1981static bool intel_set_max_freq_ratio(void)
1982{
1983	u64 base_freq, turbo_freq;
1984	u64 turbo_ratio;
1985
1986	if (slv_set_max_freq_ratio(&base_freq, &turbo_freq))
1987		goto out;
1988
1989	if (x86_match_cpu(has_glm_turbo_ratio_limits) &&
1990	    skx_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
1991		goto out;
1992
1993	if (x86_match_cpu(has_knl_turbo_ratio_limits) &&
1994	    knl_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
1995		goto out;
1996
1997	if (x86_match_cpu(has_skx_turbo_ratio_limits) &&
1998	    skx_set_max_freq_ratio(&base_freq, &turbo_freq, 4))
1999		goto out;
2000
2001	if (core_set_max_freq_ratio(&base_freq, &turbo_freq))
2002		goto out;
2003
2004	return false;
2005
2006out:
2007	/*
2008	 * Some hypervisors advertise X86_FEATURE_APERFMPERF
2009	 * but then fill all MSR's with zeroes.
2010	 * Some CPUs have turbo boost but don't declare any turbo ratio
2011	 * in MSR_TURBO_RATIO_LIMIT.
2012	 */
2013	if (!base_freq || !turbo_freq) {
2014		pr_debug("Couldn't determine cpu base or turbo frequency, necessary for scale-invariant accounting.\n");
2015		return false;
2016	}
2017
2018	turbo_ratio = div_u64(turbo_freq * SCHED_CAPACITY_SCALE, base_freq);
2019	if (!turbo_ratio) {
2020		pr_debug("Non-zero turbo and base frequencies led to a 0 ratio.\n");
2021		return false;
2022	}
2023
2024	arch_turbo_freq_ratio = turbo_ratio;
2025	arch_set_max_freq_ratio(turbo_disabled());
2026
2027	return true;
2028}
2029
2030static void init_counter_refs(void)
2031{
2032	u64 aperf, mperf;
2033
2034	rdmsrl(MSR_IA32_APERF, aperf);
2035	rdmsrl(MSR_IA32_MPERF, mperf);
2036
2037	this_cpu_write(arch_prev_aperf, aperf);
2038	this_cpu_write(arch_prev_mperf, mperf);
2039}
2040
2041static void init_freq_invariance(bool secondary)
2042{
2043	bool ret = false;
2044
2045	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
2046		return;
2047
2048	if (secondary) {
2049		if (static_branch_likely(&arch_scale_freq_key)) {
2050			init_counter_refs();
2051		}
2052		return;
2053	}
2054
2055	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2056		ret = intel_set_max_freq_ratio();
2057
2058	if (ret) {
2059		init_counter_refs();
2060		static_branch_enable(&arch_scale_freq_key);
2061	} else {
2062		pr_debug("Couldn't determine max cpu frequency, necessary for scale-invariant accounting.\n");
2063	}
2064}
2065
2066static void disable_freq_invariance_workfn(struct work_struct *work)
2067{
2068	static_branch_disable(&arch_scale_freq_key);
2069}
2070
2071static DECLARE_WORK(disable_freq_invariance_work,
2072		    disable_freq_invariance_workfn);
2073
2074DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE;
2075
2076void arch_scale_freq_tick(void)
2077{
2078	u64 freq_scale = SCHED_CAPACITY_SCALE;
2079	u64 aperf, mperf;
2080	u64 acnt, mcnt;
2081
2082	if (!arch_scale_freq_invariant())
2083		return;
2084
2085	rdmsrl(MSR_IA32_APERF, aperf);
2086	rdmsrl(MSR_IA32_MPERF, mperf);
2087
2088	acnt = aperf - this_cpu_read(arch_prev_aperf);
2089	mcnt = mperf - this_cpu_read(arch_prev_mperf);
2090
2091	this_cpu_write(arch_prev_aperf, aperf);
2092	this_cpu_write(arch_prev_mperf, mperf);
2093
2094	if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt))
2095		goto error;
2096
2097	if (check_mul_overflow(mcnt, arch_max_freq_ratio, &mcnt) || !mcnt)
2098		goto error;
2099
2100	freq_scale = div64_u64(acnt, mcnt);
2101	if (!freq_scale)
2102		goto error;
2103
2104	if (freq_scale > SCHED_CAPACITY_SCALE)
2105		freq_scale = SCHED_CAPACITY_SCALE;
2106
2107	this_cpu_write(arch_freq_scale, freq_scale);
2108	return;
2109
2110error:
2111	pr_warn("Scheduler frequency invariance went wobbly, disabling!\n");
2112	schedule_work(&disable_freq_invariance_work);
2113}
2114#else
2115static inline void init_freq_invariance(bool secondary)
2116{
2117}
2118#endif /* CONFIG_X86_64 */