1// SPDX-License-Identifier: GPL-2.0-only
   2/* cpu_feature_enabled() cannot be used this early */
   3#define USE_EARLY_PGTABLE_L5
   4
   5#include <linux/memblock.h>
   6#include <linux/linkage.h>
   7#include <linux/bitops.h>
   8#include <linux/kernel.h>
   9#include <linux/export.h>
  10#include <linux/percpu.h>
  11#include <linux/string.h>
  12#include <linux/ctype.h>
  13#include <linux/delay.h>
  14#include <linux/sched/mm.h>
  15#include <linux/sched/clock.h>
  16#include <linux/sched/task.h>
  17#include <linux/sched/smt.h>
  18#include <linux/init.h>
  19#include <linux/kprobes.h>
  20#include <linux/kgdb.h>
  21#include <linux/mem_encrypt.h>
  22#include <linux/smp.h>
  23#include <linux/cpu.h>
  24#include <linux/io.h>
  25#include <linux/syscore_ops.h>
  26#include <linux/pgtable.h>
  27#include <linux/stackprotector.h>
  28#include <linux/utsname.h>
  29
  30#include <asm/alternative.h>
  31#include <asm/cmdline.h>
  32#include <asm/perf_event.h>
  33#include <asm/mmu_context.h>
  34#include <asm/doublefault.h>
  35#include <asm/archrandom.h>
  36#include <asm/hypervisor.h>
  37#include <asm/processor.h>
  38#include <asm/tlbflush.h>
  39#include <asm/debugreg.h>
  40#include <asm/sections.h>
  41#include <asm/vsyscall.h>
  42#include <linux/topology.h>
  43#include <linux/cpumask.h>
  44#include <linux/atomic.h>
  45#include <asm/proto.h>
  46#include <asm/setup.h>
  47#include <asm/apic.h>
  48#include <asm/desc.h>
  49#include <asm/fpu/api.h>
  50#include <asm/mtrr.h>
  51#include <asm/hwcap2.h>
  52#include <linux/numa.h>
  53#include <asm/numa.h>
  54#include <asm/asm.h>
  55#include <asm/bugs.h>
  56#include <asm/cpu.h>
  57#include <asm/mce.h>
  58#include <asm/msr.h>
  59#include <asm/cacheinfo.h>
  60#include <asm/memtype.h>
  61#include <asm/microcode.h>
  62#include <asm/intel-family.h>
  63#include <asm/cpu_device_id.h>
 
  64#include <asm/uv/uv.h>
  65#include <asm/ia32.h>
  66#include <asm/set_memory.h>
  67#include <asm/traps.h>
  68#include <asm/sev.h>
  69#include <asm/tdx.h>
  70
  71#include "cpu.h"
  72
 
 
 
  73u32 elf_hwcap2 __read_mostly;
  74
  75/* Number of siblings per CPU package */
  76int smp_num_siblings = 1;
  77EXPORT_SYMBOL(smp_num_siblings);
 
 
 
 
 
 
 
 
 
 
 
 
  78
  79static struct ppin_info {
  80	int	feature;
  81	int	msr_ppin_ctl;
  82	int	msr_ppin;
  83} ppin_info[] = {
  84	[X86_VENDOR_INTEL] = {
  85		.feature = X86_FEATURE_INTEL_PPIN,
  86		.msr_ppin_ctl = MSR_PPIN_CTL,
  87		.msr_ppin = MSR_PPIN
  88	},
  89	[X86_VENDOR_AMD] = {
  90		.feature = X86_FEATURE_AMD_PPIN,
  91		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
  92		.msr_ppin = MSR_AMD_PPIN
  93	},
  94};
  95
  96static const struct x86_cpu_id ppin_cpuids[] = {
  97	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
  98	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
  99
 100	/* Legacy models without CPUID enumeration */
 101	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
 102	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 103	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
 104	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 105	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 106	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 107	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
 108	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 109	X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 110	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
 111	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
 112
 113	{}
 114};
 115
 116static void ppin_init(struct cpuinfo_x86 *c)
 117{
 118	const struct x86_cpu_id *id;
 119	unsigned long long val;
 120	struct ppin_info *info;
 121
 122	id = x86_match_cpu(ppin_cpuids);
 123	if (!id)
 124		return;
 125
 126	/*
 127	 * Testing the presence of the MSR is not enough. Need to check
 128	 * that the PPIN_CTL allows reading of the PPIN.
 129	 */
 130	info = (struct ppin_info *)id->driver_data;
 131
 132	if (rdmsrl_safe(info->msr_ppin_ctl, &val))
 133		goto clear_ppin;
 134
 135	if ((val & 3UL) == 1UL) {
 136		/* PPIN locked in disabled mode */
 137		goto clear_ppin;
 138	}
 139
 140	/* If PPIN is disabled, try to enable */
 141	if (!(val & 2UL)) {
 142		wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
 143		rdmsrl_safe(info->msr_ppin_ctl, &val);
 144	}
 145
 146	/* Is the enable bit set? */
 147	if (val & 2UL) {
 148		c->ppin = __rdmsr(info->msr_ppin);
 149		set_cpu_cap(c, info->feature);
 150		return;
 151	}
 152
 153clear_ppin:
 154	clear_cpu_cap(c, info->feature);
 155}
 156
 157static void default_init(struct cpuinfo_x86 *c)
 158{
 159#ifdef CONFIG_X86_64
 160	cpu_detect_cache_sizes(c);
 161#else
 162	/* Not much we can do here... */
 163	/* Check if at least it has cpuid */
 164	if (c->cpuid_level == -1) {
 165		/* No cpuid. It must be an ancient CPU */
 166		if (c->x86 == 4)
 167			strcpy(c->x86_model_id, "486");
 168		else if (c->x86 == 3)
 169			strcpy(c->x86_model_id, "386");
 170	}
 171#endif
 172}
 173
 174static const struct cpu_dev default_cpu = {
 175	.c_init		= default_init,
 176	.c_vendor	= "Unknown",
 177	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
 178};
 179
 180static const struct cpu_dev *this_cpu = &default_cpu;
 181
 182DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 183#ifdef CONFIG_X86_64
 184	/*
 185	 * We need valid kernel segments for data and code in long mode too
 186	 * IRET will check the segment types  kkeil 2000/10/28
 187	 * Also sysret mandates a special GDT layout
 188	 *
 189	 * TLS descriptors are currently at a different place compared to i386.
 190	 * Hopefully nobody expects them at a fixed place (Wine?)
 191	 */
 192	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 193	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
 194	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
 195	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 196	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
 197	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
 198#else
 199	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 200	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 201	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 202	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
 203	/*
 204	 * Segments used for calling PnP BIOS have byte granularity.
 205	 * They code segments and data segments have fixed 64k limits,
 206	 * the transfer segment sizes are set at run time.
 207	 */
 208	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 209	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 210	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
 211	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 212	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 213	/*
 214	 * The APM segments have byte granularity and their bases
 215	 * are set at run time.  All have 64k limits.
 216	 */
 217	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 218	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 219	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
 220
 221	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 222	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 223#endif
 224} };
 225EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 226
 227#ifdef CONFIG_X86_64
 228static int __init x86_nopcid_setup(char *s)
 229{
 230	/* nopcid doesn't accept parameters */
 231	if (s)
 232		return -EINVAL;
 233
 234	/* do not emit a message if the feature is not present */
 235	if (!boot_cpu_has(X86_FEATURE_PCID))
 236		return 0;
 237
 238	setup_clear_cpu_cap(X86_FEATURE_PCID);
 239	pr_info("nopcid: PCID feature disabled\n");
 240	return 0;
 241}
 242early_param("nopcid", x86_nopcid_setup);
 243#endif
 244
 245static int __init x86_noinvpcid_setup(char *s)
 246{
 247	/* noinvpcid doesn't accept parameters */
 248	if (s)
 249		return -EINVAL;
 250
 251	/* do not emit a message if the feature is not present */
 252	if (!boot_cpu_has(X86_FEATURE_INVPCID))
 253		return 0;
 254
 255	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
 256	pr_info("noinvpcid: INVPCID feature disabled\n");
 257	return 0;
 258}
 259early_param("noinvpcid", x86_noinvpcid_setup);
 260
 261#ifdef CONFIG_X86_32
 262static int cachesize_override = -1;
 263static int disable_x86_serial_nr = 1;
 264
 265static int __init cachesize_setup(char *str)
 266{
 267	get_option(&str, &cachesize_override);
 268	return 1;
 269}
 270__setup("cachesize=", cachesize_setup);
 271
 272/* Standard macro to see if a specific flag is changeable */
 273static inline int flag_is_changeable_p(u32 flag)
 274{
 275	u32 f1, f2;
 276
 277	/*
 278	 * Cyrix and IDT cpus allow disabling of CPUID
 279	 * so the code below may return different results
 280	 * when it is executed before and after enabling
 281	 * the CPUID. Add "volatile" to not allow gcc to
 282	 * optimize the subsequent calls to this function.
 283	 */
 284	asm volatile ("pushfl		\n\t"
 285		      "pushfl		\n\t"
 286		      "popl %0		\n\t"
 287		      "movl %0, %1	\n\t"
 288		      "xorl %2, %0	\n\t"
 289		      "pushl %0		\n\t"
 290		      "popfl		\n\t"
 291		      "pushfl		\n\t"
 292		      "popl %0		\n\t"
 293		      "popfl		\n\t"
 294
 295		      : "=&r" (f1), "=&r" (f2)
 296		      : "ir" (flag));
 297
 298	return ((f1^f2) & flag) != 0;
 299}
 300
 301/* Probe for the CPUID instruction */
 302int have_cpuid_p(void)
 303{
 304	return flag_is_changeable_p(X86_EFLAGS_ID);
 305}
 306
 307static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 308{
 309	unsigned long lo, hi;
 310
 311	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
 312		return;
 313
 314	/* Disable processor serial number: */
 315
 316	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 317	lo |= 0x200000;
 318	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 319
 320	pr_notice("CPU serial number disabled.\n");
 321	clear_cpu_cap(c, X86_FEATURE_PN);
 322
 323	/* Disabling the serial number may affect the cpuid level */
 324	c->cpuid_level = cpuid_eax(0);
 325}
 326
 327static int __init x86_serial_nr_setup(char *s)
 328{
 329	disable_x86_serial_nr = 0;
 330	return 1;
 331}
 332__setup("serialnumber", x86_serial_nr_setup);
 333#else
 334static inline int flag_is_changeable_p(u32 flag)
 335{
 336	return 1;
 337}
 338static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 339{
 340}
 341#endif
 342
 343static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 344{
 345	if (cpu_has(c, X86_FEATURE_SMEP))
 346		cr4_set_bits(X86_CR4_SMEP);
 347}
 348
 349static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 350{
 351	unsigned long eflags = native_save_fl();
 352
 353	/* This should have been cleared long ago */
 354	BUG_ON(eflags & X86_EFLAGS_AC);
 355
 356	if (cpu_has(c, X86_FEATURE_SMAP))
 357		cr4_set_bits(X86_CR4_SMAP);
 358}
 359
 360static __always_inline void setup_umip(struct cpuinfo_x86 *c)
 361{
 362	/* Check the boot processor, plus build option for UMIP. */
 363	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
 364		goto out;
 365
 366	/* Check the current processor's cpuid bits. */
 367	if (!cpu_has(c, X86_FEATURE_UMIP))
 368		goto out;
 369
 370	cr4_set_bits(X86_CR4_UMIP);
 371
 372	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
 373
 374	return;
 375
 376out:
 377	/*
 378	 * Make sure UMIP is disabled in case it was enabled in a
 379	 * previous boot (e.g., via kexec).
 380	 */
 381	cr4_clear_bits(X86_CR4_UMIP);
 382}
 383
 384/* These bits should not change their value after CPU init is finished. */
 385static const unsigned long cr4_pinned_mask =
 386	X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
 387	X86_CR4_FSGSBASE | X86_CR4_CET;
 388static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
 389static unsigned long cr4_pinned_bits __ro_after_init;
 390
 391void native_write_cr0(unsigned long val)
 392{
 393	unsigned long bits_missing = 0;
 394
 395set_register:
 396	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
 397
 398	if (static_branch_likely(&cr_pinning)) {
 399		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
 400			bits_missing = X86_CR0_WP;
 401			val |= bits_missing;
 402			goto set_register;
 403		}
 404		/* Warn after we've set the missing bits. */
 405		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
 406	}
 407}
 408EXPORT_SYMBOL(native_write_cr0);
 409
 410void __no_profile native_write_cr4(unsigned long val)
 411{
 412	unsigned long bits_changed = 0;
 413
 414set_register:
 415	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
 416
 417	if (static_branch_likely(&cr_pinning)) {
 418		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
 419			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
 420			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
 421			goto set_register;
 422		}
 423		/* Warn after we've corrected the changed bits. */
 424		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
 425			  bits_changed);
 426	}
 427}
 428#if IS_MODULE(CONFIG_LKDTM)
 429EXPORT_SYMBOL_GPL(native_write_cr4);
 430#endif
 431
 432void cr4_update_irqsoff(unsigned long set, unsigned long clear)
 433{
 434	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
 435
 436	lockdep_assert_irqs_disabled();
 437
 438	newval = (cr4 & ~clear) | set;
 439	if (newval != cr4) {
 440		this_cpu_write(cpu_tlbstate.cr4, newval);
 441		__write_cr4(newval);
 442	}
 443}
 444EXPORT_SYMBOL(cr4_update_irqsoff);
 445
 446/* Read the CR4 shadow. */
 447unsigned long cr4_read_shadow(void)
 448{
 449	return this_cpu_read(cpu_tlbstate.cr4);
 450}
 451EXPORT_SYMBOL_GPL(cr4_read_shadow);
 452
 453void cr4_init(void)
 454{
 455	unsigned long cr4 = __read_cr4();
 456
 457	if (boot_cpu_has(X86_FEATURE_PCID))
 458		cr4 |= X86_CR4_PCIDE;
 459	if (static_branch_likely(&cr_pinning))
 460		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
 461
 462	__write_cr4(cr4);
 463
 464	/* Initialize cr4 shadow for this CPU. */
 465	this_cpu_write(cpu_tlbstate.cr4, cr4);
 466}
 467
 468/*
 469 * Once CPU feature detection is finished (and boot params have been
 470 * parsed), record any of the sensitive CR bits that are set, and
 471 * enable CR pinning.
 472 */
 473static void __init setup_cr_pinning(void)
 474{
 475	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
 476	static_key_enable(&cr_pinning.key);
 477}
 478
 479static __init int x86_nofsgsbase_setup(char *arg)
 480{
 481	/* Require an exact match without trailing characters. */
 482	if (strlen(arg))
 483		return 0;
 484
 485	/* Do not emit a message if the feature is not present. */
 486	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
 487		return 1;
 488
 489	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
 490	pr_info("FSGSBASE disabled via kernel command line\n");
 491	return 1;
 492}
 493__setup("nofsgsbase", x86_nofsgsbase_setup);
 494
 495/*
 496 * Protection Keys are not available in 32-bit mode.
 497 */
 498static bool pku_disabled;
 499
 500static __always_inline void setup_pku(struct cpuinfo_x86 *c)
 501{
 502	if (c == &boot_cpu_data) {
 503		if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
 504			return;
 505		/*
 506		 * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
 507		 * bit to be set.  Enforce it.
 508		 */
 509		setup_force_cpu_cap(X86_FEATURE_OSPKE);
 510
 511	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
 512		return;
 513	}
 514
 515	cr4_set_bits(X86_CR4_PKE);
 516	/* Load the default PKRU value */
 517	pkru_write_default();
 518}
 519
 520#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 521static __init int setup_disable_pku(char *arg)
 522{
 523	/*
 524	 * Do not clear the X86_FEATURE_PKU bit.  All of the
 525	 * runtime checks are against OSPKE so clearing the
 526	 * bit does nothing.
 527	 *
 528	 * This way, we will see "pku" in cpuinfo, but not
 529	 * "ospke", which is exactly what we want.  It shows
 530	 * that the CPU has PKU, but the OS has not enabled it.
 531	 * This happens to be exactly how a system would look
 532	 * if we disabled the config option.
 533	 */
 534	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
 535	pku_disabled = true;
 536	return 1;
 537}
 538__setup("nopku", setup_disable_pku);
 539#endif
 540
 541#ifdef CONFIG_X86_KERNEL_IBT
 542
 543__noendbr u64 ibt_save(bool disable)
 544{
 545	u64 msr = 0;
 546
 547	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 548		rdmsrl(MSR_IA32_S_CET, msr);
 549		if (disable)
 550			wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
 551	}
 552
 553	return msr;
 554}
 555
 556__noendbr void ibt_restore(u64 save)
 557{
 558	u64 msr;
 559
 560	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 561		rdmsrl(MSR_IA32_S_CET, msr);
 562		msr &= ~CET_ENDBR_EN;
 563		msr |= (save & CET_ENDBR_EN);
 564		wrmsrl(MSR_IA32_S_CET, msr);
 565	}
 566}
 567
 568#endif
 569
 570static __always_inline void setup_cet(struct cpuinfo_x86 *c)
 571{
 572	bool user_shstk, kernel_ibt;
 573
 574	if (!IS_ENABLED(CONFIG_X86_CET))
 575		return;
 576
 577	kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
 578	user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
 579		     IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
 580
 581	if (!kernel_ibt && !user_shstk)
 582		return;
 583
 584	if (user_shstk)
 585		set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
 586
 587	if (kernel_ibt)
 588		wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
 589	else
 590		wrmsrl(MSR_IA32_S_CET, 0);
 591
 592	cr4_set_bits(X86_CR4_CET);
 593
 594	if (kernel_ibt && ibt_selftest()) {
 595		pr_err("IBT selftest: Failed!\n");
 596		wrmsrl(MSR_IA32_S_CET, 0);
 597		setup_clear_cpu_cap(X86_FEATURE_IBT);
 598	}
 599}
 600
 601__noendbr void cet_disable(void)
 602{
 603	if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
 604	      cpu_feature_enabled(X86_FEATURE_SHSTK)))
 605		return;
 606
 607	wrmsrl(MSR_IA32_S_CET, 0);
 608	wrmsrl(MSR_IA32_U_CET, 0);
 609}
 610
 611/*
 612 * Some CPU features depend on higher CPUID levels, which may not always
 613 * be available due to CPUID level capping or broken virtualization
 614 * software.  Add those features to this table to auto-disable them.
 615 */
 616struct cpuid_dependent_feature {
 617	u32 feature;
 618	u32 level;
 619};
 620
 621static const struct cpuid_dependent_feature
 622cpuid_dependent_features[] = {
 623	{ X86_FEATURE_MWAIT,		0x00000005 },
 624	{ X86_FEATURE_DCA,		0x00000009 },
 625	{ X86_FEATURE_XSAVE,		0x0000000d },
 626	{ 0, 0 }
 627};
 628
 629static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
 630{
 631	const struct cpuid_dependent_feature *df;
 632
 633	for (df = cpuid_dependent_features; df->feature; df++) {
 634
 635		if (!cpu_has(c, df->feature))
 636			continue;
 637		/*
 638		 * Note: cpuid_level is set to -1 if unavailable, but
 639		 * extended_extended_level is set to 0 if unavailable
 640		 * and the legitimate extended levels are all negative
 641		 * when signed; hence the weird messing around with
 642		 * signs here...
 643		 */
 644		if (!((s32)df->level < 0 ?
 645		     (u32)df->level > (u32)c->extended_cpuid_level :
 646		     (s32)df->level > (s32)c->cpuid_level))
 647			continue;
 648
 649		clear_cpu_cap(c, df->feature);
 650		if (!warn)
 651			continue;
 652
 653		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
 654			x86_cap_flag(df->feature), df->level);
 655	}
 656}
 657
 658/*
 659 * Naming convention should be: <Name> [(<Codename>)]
 660 * This table only is used unless init_<vendor>() below doesn't set it;
 661 * in particular, if CPUID levels 0x80000002..4 are supported, this
 662 * isn't used
 663 */
 664
 665/* Look up CPU names by table lookup. */
 666static const char *table_lookup_model(struct cpuinfo_x86 *c)
 667{
 668#ifdef CONFIG_X86_32
 669	const struct legacy_cpu_model_info *info;
 670
 671	if (c->x86_model >= 16)
 672		return NULL;	/* Range check */
 673
 674	if (!this_cpu)
 675		return NULL;
 676
 677	info = this_cpu->legacy_models;
 678
 679	while (info->family) {
 680		if (info->family == c->x86)
 681			return info->model_names[c->x86_model];
 682		info++;
 683	}
 684#endif
 685	return NULL;		/* Not found */
 686}
 687
 688/* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
 689__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 690__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 691
 692#ifdef CONFIG_X86_32
 693/* The 32-bit entry code needs to find cpu_entry_area. */
 694DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 695#endif
 696
 697/* Load the original GDT from the per-cpu structure */
 698void load_direct_gdt(int cpu)
 699{
 700	struct desc_ptr gdt_descr;
 701
 702	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
 703	gdt_descr.size = GDT_SIZE - 1;
 704	load_gdt(&gdt_descr);
 705}
 706EXPORT_SYMBOL_GPL(load_direct_gdt);
 707
 708/* Load a fixmap remapping of the per-cpu GDT */
 709void load_fixmap_gdt(int cpu)
 710{
 711	struct desc_ptr gdt_descr;
 712
 713	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
 714	gdt_descr.size = GDT_SIZE - 1;
 715	load_gdt(&gdt_descr);
 716}
 717EXPORT_SYMBOL_GPL(load_fixmap_gdt);
 718
 719/**
 720 * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
 721 * @cpu:	The CPU number for which this is invoked
 722 *
 723 * Invoked during early boot to switch from early GDT and early per CPU to
 724 * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
 725 * switch is implicit by loading the direct GDT. On 64bit this requires
 726 * to update GSBASE.
 727 */
 728void __init switch_gdt_and_percpu_base(int cpu)
 729{
 730	load_direct_gdt(cpu);
 731
 732#ifdef CONFIG_X86_64
 733	/*
 734	 * No need to load %gs. It is already correct.
 735	 *
 736	 * Writing %gs on 64bit would zero GSBASE which would make any per
 737	 * CPU operation up to the point of the wrmsrl() fault.
 738	 *
 739	 * Set GSBASE to the new offset. Until the wrmsrl() happens the
 740	 * early mapping is still valid. That means the GSBASE update will
 741	 * lose any prior per CPU data which was not copied over in
 742	 * setup_per_cpu_areas().
 743	 *
 744	 * This works even with stackprotector enabled because the
 745	 * per CPU stack canary is 0 in both per CPU areas.
 746	 */
 747	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
 748#else
 749	/*
 750	 * %fs is already set to __KERNEL_PERCPU, but after switching GDT
 751	 * it is required to load FS again so that the 'hidden' part is
 752	 * updated from the new GDT. Up to this point the early per CPU
 753	 * translation is active. Any content of the early per CPU data
 754	 * which was not copied over in setup_per_cpu_areas() is lost.
 755	 */
 756	loadsegment(fs, __KERNEL_PERCPU);
 757#endif
 758}
 759
 760static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
 761
 762static void get_model_name(struct cpuinfo_x86 *c)
 763{
 764	unsigned int *v;
 765	char *p, *q, *s;
 766
 767	if (c->extended_cpuid_level < 0x80000004)
 768		return;
 769
 770	v = (unsigned int *)c->x86_model_id;
 771	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
 772	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
 773	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
 774	c->x86_model_id[48] = 0;
 775
 776	/* Trim whitespace */
 777	p = q = s = &c->x86_model_id[0];
 778
 779	while (*p == ' ')
 780		p++;
 781
 782	while (*p) {
 783		/* Note the last non-whitespace index */
 784		if (!isspace(*p))
 785			s = q;
 786
 787		*q++ = *p++;
 788	}
 789
 790	*(s + 1) = '\0';
 791}
 792
 793void detect_num_cpu_cores(struct cpuinfo_x86 *c)
 794{
 795	unsigned int eax, ebx, ecx, edx;
 796
 797	c->x86_max_cores = 1;
 798	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
 799		return;
 800
 801	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
 802	if (eax & 0x1f)
 803		c->x86_max_cores = (eax >> 26) + 1;
 804}
 805
 806void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 807{
 808	unsigned int n, dummy, ebx, ecx, edx, l2size;
 809
 810	n = c->extended_cpuid_level;
 811
 812	if (n >= 0x80000005) {
 813		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
 814		c->x86_cache_size = (ecx>>24) + (edx>>24);
 815#ifdef CONFIG_X86_64
 816		/* On K8 L1 TLB is inclusive, so don't count it */
 817		c->x86_tlbsize = 0;
 818#endif
 819	}
 820
 821	if (n < 0x80000006)	/* Some chips just has a large L1. */
 822		return;
 823
 824	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
 825	l2size = ecx >> 16;
 826
 827#ifdef CONFIG_X86_64
 828	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
 829#else
 830	/* do processor-specific cache resizing */
 831	if (this_cpu->legacy_cache_size)
 832		l2size = this_cpu->legacy_cache_size(c, l2size);
 833
 834	/* Allow user to override all this if necessary. */
 835	if (cachesize_override != -1)
 836		l2size = cachesize_override;
 837
 838	if (l2size == 0)
 839		return;		/* Again, no L2 cache is possible */
 840#endif
 841
 842	c->x86_cache_size = l2size;
 843}
 844
 845u16 __read_mostly tlb_lli_4k[NR_INFO];
 846u16 __read_mostly tlb_lli_2m[NR_INFO];
 847u16 __read_mostly tlb_lli_4m[NR_INFO];
 848u16 __read_mostly tlb_lld_4k[NR_INFO];
 849u16 __read_mostly tlb_lld_2m[NR_INFO];
 850u16 __read_mostly tlb_lld_4m[NR_INFO];
 851u16 __read_mostly tlb_lld_1g[NR_INFO];
 852
 853static void cpu_detect_tlb(struct cpuinfo_x86 *c)
 854{
 855	if (this_cpu->c_detect_tlb)
 856		this_cpu->c_detect_tlb(c);
 857
 858	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
 859		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 860		tlb_lli_4m[ENTRIES]);
 861
 862	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
 863		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
 864		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
 865}
 866
 867int detect_ht_early(struct cpuinfo_x86 *c)
 868{
 869#ifdef CONFIG_SMP
 870	u32 eax, ebx, ecx, edx;
 871
 872	if (!cpu_has(c, X86_FEATURE_HT))
 873		return -1;
 874
 875	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
 876		return -1;
 877
 878	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
 879		return -1;
 880
 881	cpuid(1, &eax, &ebx, &ecx, &edx);
 882
 883	smp_num_siblings = (ebx & 0xff0000) >> 16;
 884	if (smp_num_siblings == 1)
 885		pr_info_once("CPU0: Hyper-Threading is disabled\n");
 886#endif
 887	return 0;
 888}
 889
 890void detect_ht(struct cpuinfo_x86 *c)
 891{
 892#ifdef CONFIG_SMP
 893	int index_msb, core_bits;
 894
 895	if (detect_ht_early(c) < 0)
 896		return;
 897
 898	index_msb = get_count_order(smp_num_siblings);
 899	c->topo.pkg_id = apic->phys_pkg_id(c->topo.initial_apicid, index_msb);
 900
 901	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
 902
 903	index_msb = get_count_order(smp_num_siblings);
 904
 905	core_bits = get_count_order(c->x86_max_cores);
 906
 907	c->topo.core_id = apic->phys_pkg_id(c->topo.initial_apicid, index_msb) &
 908		((1 << core_bits) - 1);
 909#endif
 910}
 911
 912static void get_cpu_vendor(struct cpuinfo_x86 *c)
 913{
 914	char *v = c->x86_vendor_id;
 915	int i;
 916
 917	for (i = 0; i < X86_VENDOR_NUM; i++) {
 918		if (!cpu_devs[i])
 919			break;
 920
 921		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
 922		    (cpu_devs[i]->c_ident[1] &&
 923		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
 924
 925			this_cpu = cpu_devs[i];
 926			c->x86_vendor = this_cpu->c_x86_vendor;
 927			return;
 928		}
 929	}
 930
 931	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
 932		    "CPU: Your system may be unstable.\n", v);
 933
 934	c->x86_vendor = X86_VENDOR_UNKNOWN;
 935	this_cpu = &default_cpu;
 936}
 937
 938void cpu_detect(struct cpuinfo_x86 *c)
 939{
 940	/* Get vendor name */
 941	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
 942	      (unsigned int *)&c->x86_vendor_id[0],
 943	      (unsigned int *)&c->x86_vendor_id[8],
 944	      (unsigned int *)&c->x86_vendor_id[4]);
 945
 946	c->x86 = 4;
 947	/* Intel-defined flags: level 0x00000001 */
 948	if (c->cpuid_level >= 0x00000001) {
 949		u32 junk, tfms, cap0, misc;
 950
 951		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
 952		c->x86		= x86_family(tfms);
 953		c->x86_model	= x86_model(tfms);
 954		c->x86_stepping	= x86_stepping(tfms);
 955
 956		if (cap0 & (1<<19)) {
 957			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
 958			c->x86_cache_alignment = c->x86_clflush_size;
 959		}
 960	}
 961}
 962
 963static void apply_forced_caps(struct cpuinfo_x86 *c)
 964{
 965	int i;
 966
 967	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
 968		c->x86_capability[i] &= ~cpu_caps_cleared[i];
 969		c->x86_capability[i] |= cpu_caps_set[i];
 970	}
 971}
 972
 973static void init_speculation_control(struct cpuinfo_x86 *c)
 974{
 975	/*
 976	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
 977	 * and they also have a different bit for STIBP support. Also,
 978	 * a hypervisor might have set the individual AMD bits even on
 979	 * Intel CPUs, for finer-grained selection of what's available.
 980	 */
 981	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
 982		set_cpu_cap(c, X86_FEATURE_IBRS);
 983		set_cpu_cap(c, X86_FEATURE_IBPB);
 984		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 985	}
 986
 987	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
 988		set_cpu_cap(c, X86_FEATURE_STIBP);
 989
 990	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
 991	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
 992		set_cpu_cap(c, X86_FEATURE_SSBD);
 993
 994	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
 995		set_cpu_cap(c, X86_FEATURE_IBRS);
 996		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 997	}
 998
 999	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
1000		set_cpu_cap(c, X86_FEATURE_IBPB);
1001
1002	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
1003		set_cpu_cap(c, X86_FEATURE_STIBP);
1004		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
1005	}
1006
1007	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
1008		set_cpu_cap(c, X86_FEATURE_SSBD);
1009		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
1010		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
1011	}
1012}
1013
1014void get_cpu_cap(struct cpuinfo_x86 *c)
1015{
1016	u32 eax, ebx, ecx, edx;
1017
1018	/* Intel-defined flags: level 0x00000001 */
1019	if (c->cpuid_level >= 0x00000001) {
1020		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
1021
1022		c->x86_capability[CPUID_1_ECX] = ecx;
1023		c->x86_capability[CPUID_1_EDX] = edx;
1024	}
1025
1026	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
1027	if (c->cpuid_level >= 0x00000006)
1028		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
1029
1030	/* Additional Intel-defined flags: level 0x00000007 */
1031	if (c->cpuid_level >= 0x00000007) {
1032		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
1033		c->x86_capability[CPUID_7_0_EBX] = ebx;
1034		c->x86_capability[CPUID_7_ECX] = ecx;
1035		c->x86_capability[CPUID_7_EDX] = edx;
1036
1037		/* Check valid sub-leaf index before accessing it */
1038		if (eax >= 1) {
1039			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
1040			c->x86_capability[CPUID_7_1_EAX] = eax;
1041		}
1042	}
1043
1044	/* Extended state features: level 0x0000000d */
1045	if (c->cpuid_level >= 0x0000000d) {
1046		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1047
1048		c->x86_capability[CPUID_D_1_EAX] = eax;
1049	}
1050
1051	/* AMD-defined flags: level 0x80000001 */
1052	eax = cpuid_eax(0x80000000);
1053	c->extended_cpuid_level = eax;
1054
1055	if ((eax & 0xffff0000) == 0x80000000) {
1056		if (eax >= 0x80000001) {
1057			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1058
1059			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1060			c->x86_capability[CPUID_8000_0001_EDX] = edx;
1061		}
1062	}
1063
1064	if (c->extended_cpuid_level >= 0x80000007) {
1065		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1066
1067		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1068		c->x86_power = edx;
1069	}
1070
1071	if (c->extended_cpuid_level >= 0x80000008) {
1072		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1073		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1074	}
1075
1076	if (c->extended_cpuid_level >= 0x8000000a)
1077		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1078
1079	if (c->extended_cpuid_level >= 0x8000001f)
1080		c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1081
1082	if (c->extended_cpuid_level >= 0x80000021)
1083		c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1084
1085	init_scattered_cpuid_features(c);
1086	init_speculation_control(c);
1087
1088	/*
1089	 * Clear/Set all flags overridden by options, after probe.
1090	 * This needs to happen each time we re-probe, which may happen
1091	 * several times during CPU initialization.
1092	 */
1093	apply_forced_caps(c);
1094}
1095
1096void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1097{
1098	u32 eax, ebx, ecx, edx;
1099	bool vp_bits_from_cpuid = true;
1100
1101	if (!cpu_has(c, X86_FEATURE_CPUID) ||
1102	    (c->extended_cpuid_level < 0x80000008))
1103		vp_bits_from_cpuid = false;
1104
1105	if (vp_bits_from_cpuid) {
1106		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1107
1108		c->x86_virt_bits = (eax >> 8) & 0xff;
1109		c->x86_phys_bits = eax & 0xff;
1110	} else {
1111		if (IS_ENABLED(CONFIG_X86_64)) {
1112			c->x86_clflush_size = 64;
1113			c->x86_phys_bits = 36;
1114			c->x86_virt_bits = 48;
1115		} else {
1116			c->x86_clflush_size = 32;
1117			c->x86_virt_bits = 32;
1118			c->x86_phys_bits = 32;
1119
1120			if (cpu_has(c, X86_FEATURE_PAE) ||
1121			    cpu_has(c, X86_FEATURE_PSE36))
1122				c->x86_phys_bits = 36;
1123		}
1124	}
1125	c->x86_cache_bits = c->x86_phys_bits;
1126	c->x86_cache_alignment = c->x86_clflush_size;
1127}
1128
1129static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1130{
1131#ifdef CONFIG_X86_32
1132	int i;
1133
1134	/*
1135	 * First of all, decide if this is a 486 or higher
1136	 * It's a 486 if we can modify the AC flag
1137	 */
1138	if (flag_is_changeable_p(X86_EFLAGS_AC))
1139		c->x86 = 4;
1140	else
1141		c->x86 = 3;
1142
1143	for (i = 0; i < X86_VENDOR_NUM; i++)
1144		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1145			c->x86_vendor_id[0] = 0;
1146			cpu_devs[i]->c_identify(c);
1147			if (c->x86_vendor_id[0]) {
1148				get_cpu_vendor(c);
1149				break;
1150			}
1151		}
1152#endif
1153}
1154
1155#define NO_SPECULATION		BIT(0)
1156#define NO_MELTDOWN		BIT(1)
1157#define NO_SSB			BIT(2)
1158#define NO_L1TF			BIT(3)
1159#define NO_MDS			BIT(4)
1160#define MSBDS_ONLY		BIT(5)
1161#define NO_SWAPGS		BIT(6)
1162#define NO_ITLB_MULTIHIT	BIT(7)
1163#define NO_SPECTRE_V2		BIT(8)
1164#define NO_MMIO			BIT(9)
1165#define NO_EIBRS_PBRSB		BIT(10)
 
1166
1167#define VULNWL(vendor, family, model, whitelist)	\
1168	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1169
1170#define VULNWL_INTEL(model, whitelist)		\
1171	VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1172
1173#define VULNWL_AMD(family, whitelist)		\
1174	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1175
1176#define VULNWL_HYGON(family, whitelist)		\
1177	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1178
1179static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1180	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
1181	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
1182	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
1183	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),
1184	VULNWL(VORTEX,	5, X86_MODEL_ANY,	NO_SPECULATION),
1185	VULNWL(VORTEX,	6, X86_MODEL_ANY,	NO_SPECULATION),
1186
1187	/* Intel Family 6 */
1188	VULNWL_INTEL(TIGERLAKE,			NO_MMIO),
1189	VULNWL_INTEL(TIGERLAKE_L,		NO_MMIO),
1190	VULNWL_INTEL(ALDERLAKE,			NO_MMIO),
1191	VULNWL_INTEL(ALDERLAKE_L,		NO_MMIO),
1192
1193	VULNWL_INTEL(ATOM_SALTWELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1194	VULNWL_INTEL(ATOM_SALTWELL_TABLET,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1195	VULNWL_INTEL(ATOM_SALTWELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1196	VULNWL_INTEL(ATOM_BONNELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1197	VULNWL_INTEL(ATOM_BONNELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1198
1199	VULNWL_INTEL(ATOM_SILVERMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1200	VULNWL_INTEL(ATOM_SILVERMONT_D,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1201	VULNWL_INTEL(ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1202	VULNWL_INTEL(ATOM_AIRMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1203	VULNWL_INTEL(XEON_PHI_KNL,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1204	VULNWL_INTEL(XEON_PHI_KNM,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1205
1206	VULNWL_INTEL(CORE_YONAH,		NO_SSB),
1207
1208	VULNWL_INTEL(ATOM_AIRMONT_MID,		NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1209	VULNWL_INTEL(ATOM_AIRMONT_NP,		NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1210
1211	VULNWL_INTEL(ATOM_GOLDMONT,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1212	VULNWL_INTEL(ATOM_GOLDMONT_D,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1213	VULNWL_INTEL(ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1214
1215	/*
1216	 * Technically, swapgs isn't serializing on AMD (despite it previously
1217	 * being documented as such in the APM).  But according to AMD, %gs is
1218	 * updated non-speculatively, and the issuing of %gs-relative memory
1219	 * operands will be blocked until the %gs update completes, which is
1220	 * good enough for our purposes.
1221	 */
1222
1223	VULNWL_INTEL(ATOM_TREMONT,		NO_EIBRS_PBRSB),
1224	VULNWL_INTEL(ATOM_TREMONT_L,		NO_EIBRS_PBRSB),
1225	VULNWL_INTEL(ATOM_TREMONT_D,		NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1226
1227	/* AMD Family 0xf - 0x12 */
1228	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1229	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1230	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1231	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1232
1233	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1234	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1235	VULNWL_HYGON(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1236
1237	/* Zhaoxin Family 7 */
1238	VULNWL(CENTAUR,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1239	VULNWL(ZHAOXIN,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1240	{}
1241};
1242
1243#define VULNBL(vendor, family, model, blacklist)	\
1244	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1245
1246#define VULNBL_INTEL_STEPPINGS(model, steppings, issues)		   \
1247	X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6,		   \
1248					    INTEL_FAM6_##model, steppings, \
1249					    X86_FEATURE_ANY, issues)
1250
1251#define VULNBL_AMD(family, blacklist)		\
1252	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1253
1254#define VULNBL_HYGON(family, blacklist)		\
1255	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1256
1257#define SRBDS		BIT(0)
1258/* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1259#define MMIO		BIT(1)
1260/* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1261#define MMIO_SBDS	BIT(2)
1262/* CPU is affected by RETbleed, speculating where you would not expect it */
1263#define RETBLEED	BIT(3)
1264/* CPU is affected by SMT (cross-thread) return predictions */
1265#define SMT_RSB		BIT(4)
1266/* CPU is affected by SRSO */
1267#define SRSO		BIT(5)
1268/* CPU is affected by GDS */
1269#define GDS		BIT(6)
 
 
1270
1271static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1272	VULNBL_INTEL_STEPPINGS(IVYBRIDGE,	X86_STEPPING_ANY,		SRBDS),
1273	VULNBL_INTEL_STEPPINGS(HASWELL,		X86_STEPPING_ANY,		SRBDS),
1274	VULNBL_INTEL_STEPPINGS(HASWELL_L,	X86_STEPPING_ANY,		SRBDS),
1275	VULNBL_INTEL_STEPPINGS(HASWELL_G,	X86_STEPPING_ANY,		SRBDS),
1276	VULNBL_INTEL_STEPPINGS(HASWELL_X,	X86_STEPPING_ANY,		MMIO),
1277	VULNBL_INTEL_STEPPINGS(BROADWELL_D,	X86_STEPPING_ANY,		MMIO),
1278	VULNBL_INTEL_STEPPINGS(BROADWELL_G,	X86_STEPPING_ANY,		SRBDS),
1279	VULNBL_INTEL_STEPPINGS(BROADWELL_X,	X86_STEPPING_ANY,		MMIO),
1280	VULNBL_INTEL_STEPPINGS(BROADWELL,	X86_STEPPING_ANY,		SRBDS),
1281	VULNBL_INTEL_STEPPINGS(SKYLAKE_X,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS),
1282	VULNBL_INTEL_STEPPINGS(SKYLAKE_L,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1283	VULNBL_INTEL_STEPPINGS(SKYLAKE,		X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1284	VULNBL_INTEL_STEPPINGS(KABYLAKE_L,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1285	VULNBL_INTEL_STEPPINGS(KABYLAKE,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1286	VULNBL_INTEL_STEPPINGS(CANNONLAKE_L,	X86_STEPPING_ANY,		RETBLEED),
1287	VULNBL_INTEL_STEPPINGS(ICELAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1288	VULNBL_INTEL_STEPPINGS(ICELAKE_D,	X86_STEPPING_ANY,		MMIO | GDS),
1289	VULNBL_INTEL_STEPPINGS(ICELAKE_X,	X86_STEPPING_ANY,		MMIO | GDS),
1290	VULNBL_INTEL_STEPPINGS(COMETLAKE,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1291	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPINGS(0x0, 0x0),	MMIO | RETBLEED),
1292	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1293	VULNBL_INTEL_STEPPINGS(TIGERLAKE_L,	X86_STEPPING_ANY,		GDS),
1294	VULNBL_INTEL_STEPPINGS(TIGERLAKE,	X86_STEPPING_ANY,		GDS),
1295	VULNBL_INTEL_STEPPINGS(LAKEFIELD,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1296	VULNBL_INTEL_STEPPINGS(ROCKETLAKE,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS),
1297	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS),
1298	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D,	X86_STEPPING_ANY,		MMIO),
1299	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS),
 
 
 
 
 
 
 
 
 
1300
1301	VULNBL_AMD(0x15, RETBLEED),
1302	VULNBL_AMD(0x16, RETBLEED),
1303	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1304	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1305	VULNBL_AMD(0x19, SRSO),
1306	{}
1307};
1308
1309static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1310{
1311	const struct x86_cpu_id *m = x86_match_cpu(table);
1312
1313	return m && !!(m->driver_data & which);
1314}
1315
1316u64 x86_read_arch_cap_msr(void)
1317{
1318	u64 ia32_cap = 0;
1319
1320	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1321		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
1322
1323	return ia32_cap;
1324}
1325
1326static bool arch_cap_mmio_immune(u64 ia32_cap)
1327{
1328	return (ia32_cap & ARCH_CAP_FBSDP_NO &&
1329		ia32_cap & ARCH_CAP_PSDP_NO &&
1330		ia32_cap & ARCH_CAP_SBDR_SSDP_NO);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1331}
1332
1333static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1334{
1335	u64 ia32_cap = x86_read_arch_cap_msr();
1336
1337	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1338	if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1339	    !(ia32_cap & ARCH_CAP_PSCHANGE_MC_NO))
1340		setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1341
1342	if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1343		return;
1344
1345	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1346
1347	if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1348		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1349
1350	if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1351	    !(ia32_cap & ARCH_CAP_SSB_NO) &&
1352	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1353		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1354
1355	/*
1356	 * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1357	 * flag and protect from vendor-specific bugs via the whitelist.
 
 
 
1358	 */
1359	if ((ia32_cap & ARCH_CAP_IBRS_ALL) || cpu_has(c, X86_FEATURE_AUTOIBRS)) {
 
 
1360		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1361		if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1362		    !(ia32_cap & ARCH_CAP_PBRSB_NO))
1363			setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1364	}
1365
1366	if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1367	    !(ia32_cap & ARCH_CAP_MDS_NO)) {
1368		setup_force_cpu_bug(X86_BUG_MDS);
1369		if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1370			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1371	}
1372
1373	if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1374		setup_force_cpu_bug(X86_BUG_SWAPGS);
1375
1376	/*
1377	 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1378	 *	- TSX is supported or
1379	 *	- TSX_CTRL is present
1380	 *
1381	 * TSX_CTRL check is needed for cases when TSX could be disabled before
1382	 * the kernel boot e.g. kexec.
1383	 * TSX_CTRL check alone is not sufficient for cases when the microcode
1384	 * update is not present or running as guest that don't get TSX_CTRL.
1385	 */
1386	if (!(ia32_cap & ARCH_CAP_TAA_NO) &&
1387	    (cpu_has(c, X86_FEATURE_RTM) ||
1388	     (ia32_cap & ARCH_CAP_TSX_CTRL_MSR)))
1389		setup_force_cpu_bug(X86_BUG_TAA);
1390
1391	/*
1392	 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1393	 * in the vulnerability blacklist.
1394	 *
1395	 * Some of the implications and mitigation of Shared Buffers Data
1396	 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1397	 * SRBDS.
1398	 */
1399	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1400	     cpu_has(c, X86_FEATURE_RDSEED)) &&
1401	    cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1402		    setup_force_cpu_bug(X86_BUG_SRBDS);
1403
1404	/*
1405	 * Processor MMIO Stale Data bug enumeration
1406	 *
1407	 * Affected CPU list is generally enough to enumerate the vulnerability,
1408	 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1409	 * not want the guest to enumerate the bug.
1410	 *
1411	 * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1412	 * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1413	 */
1414	if (!arch_cap_mmio_immune(ia32_cap)) {
1415		if (cpu_matches(cpu_vuln_blacklist, MMIO))
1416			setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1417		else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1418			setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1419	}
1420
1421	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1422		if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA))
1423			setup_force_cpu_bug(X86_BUG_RETBLEED);
1424	}
1425
1426	if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1427		setup_force_cpu_bug(X86_BUG_SMT_RSB);
1428
1429	if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1430		if (cpu_matches(cpu_vuln_blacklist, SRSO))
1431			setup_force_cpu_bug(X86_BUG_SRSO);
1432	}
1433
1434	/*
1435	 * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1436	 * an affected processor, the VMM may have disabled the use of GATHER by
1437	 * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1438	 * which means that AVX will be disabled.
1439	 */
1440	if (cpu_matches(cpu_vuln_blacklist, GDS) && !(ia32_cap & ARCH_CAP_GDS_NO) &&
1441	    boot_cpu_has(X86_FEATURE_AVX))
1442		setup_force_cpu_bug(X86_BUG_GDS);
1443
 
 
 
 
 
 
 
 
 
 
1444	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1445		return;
1446
1447	/* Rogue Data Cache Load? No! */
1448	if (ia32_cap & ARCH_CAP_RDCL_NO)
1449		return;
1450
1451	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1452
1453	if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1454		return;
1455
1456	setup_force_cpu_bug(X86_BUG_L1TF);
1457}
1458
1459/*
1460 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1461 * unfortunately, that's not true in practice because of early VIA
1462 * chips and (more importantly) broken virtualizers that are not easy
1463 * to detect. In the latter case it doesn't even *fail* reliably, so
1464 * probing for it doesn't even work. Disable it completely on 32-bit
1465 * unless we can find a reliable way to detect all the broken cases.
1466 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1467 */
1468static void detect_nopl(void)
1469{
1470#ifdef CONFIG_X86_32
1471	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1472#else
1473	setup_force_cpu_cap(X86_FEATURE_NOPL);
1474#endif
1475}
1476
1477/*
1478 * We parse cpu parameters early because fpu__init_system() is executed
1479 * before parse_early_param().
1480 */
1481static void __init cpu_parse_early_param(void)
1482{
1483	char arg[128];
1484	char *argptr = arg, *opt;
1485	int arglen, taint = 0;
1486
1487#ifdef CONFIG_X86_32
1488	if (cmdline_find_option_bool(boot_command_line, "no387"))
1489#ifdef CONFIG_MATH_EMULATION
1490		setup_clear_cpu_cap(X86_FEATURE_FPU);
1491#else
1492		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1493#endif
1494
1495	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1496		setup_clear_cpu_cap(X86_FEATURE_FXSR);
1497#endif
1498
1499	if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1500		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1501
1502	if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1503		setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1504
1505	if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1506		setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1507
1508	if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1509		setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1510
1511	arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1512	if (arglen <= 0)
1513		return;
1514
1515	pr_info("Clearing CPUID bits:");
1516
1517	while (argptr) {
1518		bool found __maybe_unused = false;
1519		unsigned int bit;
1520
1521		opt = strsep(&argptr, ",");
1522
1523		/*
1524		 * Handle naked numbers first for feature flags which don't
1525		 * have names.
1526		 */
1527		if (!kstrtouint(opt, 10, &bit)) {
1528			if (bit < NCAPINTS * 32) {
1529
1530				/* empty-string, i.e., ""-defined feature flags */
1531				if (!x86_cap_flags[bit])
1532					pr_cont(" " X86_CAP_FMT_NUM, x86_cap_flag_num(bit));
1533				else
1534					pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1535
1536				setup_clear_cpu_cap(bit);
1537				taint++;
1538			}
1539			/*
1540			 * The assumption is that there are no feature names with only
1541			 * numbers in the name thus go to the next argument.
1542			 */
1543			continue;
1544		}
1545
1546		for (bit = 0; bit < 32 * NCAPINTS; bit++) {
1547			if (!x86_cap_flag(bit))
1548				continue;
1549
1550			if (strcmp(x86_cap_flag(bit), opt))
1551				continue;
1552
1553			pr_cont(" %s", opt);
1554			setup_clear_cpu_cap(bit);
1555			taint++;
1556			found = true;
1557			break;
1558		}
1559
1560		if (!found)
1561			pr_cont(" (unknown: %s)", opt);
1562	}
1563	pr_cont("\n");
1564
1565	if (taint)
1566		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1567}
1568
1569/*
1570 * Do minimum CPU detection early.
1571 * Fields really needed: vendor, cpuid_level, family, model, mask,
1572 * cache alignment.
1573 * The others are not touched to avoid unwanted side effects.
1574 *
1575 * WARNING: this function is only called on the boot CPU.  Don't add code
1576 * here that is supposed to run on all CPUs.
1577 */
1578static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1579{
1580	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1581	c->extended_cpuid_level = 0;
1582
1583	if (!have_cpuid_p())
1584		identify_cpu_without_cpuid(c);
1585
1586	/* cyrix could have cpuid enabled via c_identify()*/
1587	if (have_cpuid_p()) {
1588		cpu_detect(c);
1589		get_cpu_vendor(c);
 
1590		get_cpu_cap(c);
1591		setup_force_cpu_cap(X86_FEATURE_CPUID);
1592		get_cpu_address_sizes(c);
1593		cpu_parse_early_param();
1594
 
 
1595		if (this_cpu->c_early_init)
1596			this_cpu->c_early_init(c);
1597
1598		c->cpu_index = 0;
1599		filter_cpuid_features(c, false);
1600
1601		if (this_cpu->c_bsp_init)
1602			this_cpu->c_bsp_init(c);
1603	} else {
1604		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1605		get_cpu_address_sizes(c);
 
1606	}
1607
1608	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1609
1610	cpu_set_bug_bits(c);
1611
1612	sld_setup(c);
1613
1614#ifdef CONFIG_X86_32
1615	/*
1616	 * Regardless of whether PCID is enumerated, the SDM says
1617	 * that it can't be enabled in 32-bit mode.
1618	 */
1619	setup_clear_cpu_cap(X86_FEATURE_PCID);
1620#endif
1621
1622	/*
1623	 * Later in the boot process pgtable_l5_enabled() relies on
1624	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1625	 * enabled by this point we need to clear the feature bit to avoid
1626	 * false-positives at the later stage.
1627	 *
1628	 * pgtable_l5_enabled() can be false here for several reasons:
1629	 *  - 5-level paging is disabled compile-time;
1630	 *  - it's 32-bit kernel;
1631	 *  - machine doesn't support 5-level paging;
1632	 *  - user specified 'no5lvl' in kernel command line.
1633	 */
1634	if (!pgtable_l5_enabled())
1635		setup_clear_cpu_cap(X86_FEATURE_LA57);
1636
1637	detect_nopl();
1638}
1639
1640void __init early_cpu_init(void)
1641{
1642	const struct cpu_dev *const *cdev;
1643	int count = 0;
1644
1645#ifdef CONFIG_PROCESSOR_SELECT
1646	pr_info("KERNEL supported cpus:\n");
1647#endif
1648
1649	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1650		const struct cpu_dev *cpudev = *cdev;
1651
1652		if (count >= X86_VENDOR_NUM)
1653			break;
1654		cpu_devs[count] = cpudev;
1655		count++;
1656
1657#ifdef CONFIG_PROCESSOR_SELECT
1658		{
1659			unsigned int j;
1660
1661			for (j = 0; j < 2; j++) {
1662				if (!cpudev->c_ident[j])
1663					continue;
1664				pr_info("  %s %s\n", cpudev->c_vendor,
1665					cpudev->c_ident[j]);
1666			}
1667		}
1668#endif
1669	}
1670	early_identify_cpu(&boot_cpu_data);
1671}
1672
1673static bool detect_null_seg_behavior(void)
1674{
1675	/*
1676	 * Empirically, writing zero to a segment selector on AMD does
1677	 * not clear the base, whereas writing zero to a segment
1678	 * selector on Intel does clear the base.  Intel's behavior
1679	 * allows slightly faster context switches in the common case
1680	 * where GS is unused by the prev and next threads.
1681	 *
1682	 * Since neither vendor documents this anywhere that I can see,
1683	 * detect it directly instead of hard-coding the choice by
1684	 * vendor.
1685	 *
1686	 * I've designated AMD's behavior as the "bug" because it's
1687	 * counterintuitive and less friendly.
1688	 */
1689
1690	unsigned long old_base, tmp;
1691	rdmsrl(MSR_FS_BASE, old_base);
1692	wrmsrl(MSR_FS_BASE, 1);
1693	loadsegment(fs, 0);
1694	rdmsrl(MSR_FS_BASE, tmp);
1695	wrmsrl(MSR_FS_BASE, old_base);
1696	return tmp == 0;
1697}
1698
1699void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1700{
1701	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1702	if (!IS_ENABLED(CONFIG_X86_64))
1703		return;
1704
1705	if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
1706		return;
1707
1708	/*
1709	 * CPUID bit above wasn't set. If this kernel is still running
1710	 * as a HV guest, then the HV has decided not to advertize
1711	 * that CPUID bit for whatever reason.	For example, one
1712	 * member of the migration pool might be vulnerable.  Which
1713	 * means, the bug is present: set the BUG flag and return.
1714	 */
1715	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1716		set_cpu_bug(c, X86_BUG_NULL_SEG);
1717		return;
1718	}
1719
1720	/*
1721	 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1722	 * 0x18 is the respective family for Hygon.
1723	 */
1724	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1725	    detect_null_seg_behavior())
1726		return;
1727
1728	/* All the remaining ones are affected */
1729	set_cpu_bug(c, X86_BUG_NULL_SEG);
1730}
1731
1732static void generic_identify(struct cpuinfo_x86 *c)
1733{
1734	c->extended_cpuid_level = 0;
1735
1736	if (!have_cpuid_p())
1737		identify_cpu_without_cpuid(c);
1738
1739	/* cyrix could have cpuid enabled via c_identify()*/
1740	if (!have_cpuid_p())
1741		return;
1742
1743	cpu_detect(c);
1744
1745	get_cpu_vendor(c);
1746
1747	get_cpu_cap(c);
1748
1749	get_cpu_address_sizes(c);
1750
1751	if (c->cpuid_level >= 0x00000001) {
1752		c->topo.initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1753#ifdef CONFIG_X86_32
1754# ifdef CONFIG_SMP
1755		c->topo.apicid = apic->phys_pkg_id(c->topo.initial_apicid, 0);
1756# else
1757		c->topo.apicid = c->topo.initial_apicid;
1758# endif
1759#endif
1760		c->topo.pkg_id = c->topo.initial_apicid;
1761	}
1762
1763	get_model_name(c); /* Default name */
1764
1765	/*
1766	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1767	 * systems that run Linux at CPL > 0 may or may not have the
1768	 * issue, but, even if they have the issue, there's absolutely
1769	 * nothing we can do about it because we can't use the real IRET
1770	 * instruction.
1771	 *
1772	 * NB: For the time being, only 32-bit kernels support
1773	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1774	 * whether to apply espfix using paravirt hooks.  If any
1775	 * non-paravirt system ever shows up that does *not* have the
1776	 * ESPFIX issue, we can change this.
1777	 */
1778#ifdef CONFIG_X86_32
1779	set_cpu_bug(c, X86_BUG_ESPFIX);
1780#endif
1781}
1782
1783/*
1784 * Validate that ACPI/mptables have the same information about the
1785 * effective APIC id and update the package map.
1786 */
1787static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1788{
1789#ifdef CONFIG_SMP
1790	unsigned int cpu = smp_processor_id();
1791	u32 apicid;
1792
1793	apicid = apic->cpu_present_to_apicid(cpu);
1794
1795	if (apicid != c->topo.apicid) {
1796		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1797		       cpu, apicid, c->topo.initial_apicid);
1798	}
1799	BUG_ON(topology_update_package_map(c->topo.pkg_id, cpu));
1800	BUG_ON(topology_update_die_map(c->topo.die_id, cpu));
1801#else
1802	c->topo.logical_pkg_id = 0;
1803#endif
1804}
1805
1806/*
1807 * This does the hard work of actually picking apart the CPU stuff...
1808 */
1809static void identify_cpu(struct cpuinfo_x86 *c)
1810{
1811	int i;
1812
1813	c->loops_per_jiffy = loops_per_jiffy;
1814	c->x86_cache_size = 0;
1815	c->x86_vendor = X86_VENDOR_UNKNOWN;
1816	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
1817	c->x86_vendor_id[0] = '\0'; /* Unset */
1818	c->x86_model_id[0] = '\0';  /* Unset */
1819	c->x86_max_cores = 1;
1820	c->x86_coreid_bits = 0;
1821	c->topo.cu_id = 0xff;
1822	c->topo.llc_id = BAD_APICID;
1823	c->topo.l2c_id = BAD_APICID;
1824#ifdef CONFIG_X86_64
1825	c->x86_clflush_size = 64;
1826	c->x86_phys_bits = 36;
1827	c->x86_virt_bits = 48;
1828#else
1829	c->cpuid_level = -1;	/* CPUID not detected */
1830	c->x86_clflush_size = 32;
1831	c->x86_phys_bits = 32;
1832	c->x86_virt_bits = 32;
1833#endif
1834	c->x86_cache_alignment = c->x86_clflush_size;
1835	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1836#ifdef CONFIG_X86_VMX_FEATURE_NAMES
1837	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1838#endif
1839
1840	generic_identify(c);
1841
 
 
1842	if (this_cpu->c_identify)
1843		this_cpu->c_identify(c);
1844
1845	/* Clear/Set all flags overridden by options, after probe */
1846	apply_forced_caps(c);
1847
1848#ifdef CONFIG_X86_64
1849	c->topo.apicid = apic->phys_pkg_id(c->topo.initial_apicid, 0);
1850#endif
1851
1852
1853	/*
1854	 * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1855	 * Hygon will clear it in ->c_init() below.
1856	 */
1857	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1858
1859	/*
1860	 * Vendor-specific initialization.  In this section we
1861	 * canonicalize the feature flags, meaning if there are
1862	 * features a certain CPU supports which CPUID doesn't
1863	 * tell us, CPUID claiming incorrect flags, or other bugs,
1864	 * we handle them here.
1865	 *
1866	 * At the end of this section, c->x86_capability better
1867	 * indicate the features this CPU genuinely supports!
1868	 */
1869	if (this_cpu->c_init)
1870		this_cpu->c_init(c);
1871
1872	/* Disable the PN if appropriate */
1873	squash_the_stupid_serial_number(c);
1874
1875	/* Set up SMEP/SMAP/UMIP */
1876	setup_smep(c);
1877	setup_smap(c);
1878	setup_umip(c);
1879
1880	/* Enable FSGSBASE instructions if available. */
1881	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1882		cr4_set_bits(X86_CR4_FSGSBASE);
1883		elf_hwcap2 |= HWCAP2_FSGSBASE;
1884	}
1885
1886	/*
1887	 * The vendor-specific functions might have changed features.
1888	 * Now we do "generic changes."
1889	 */
1890
1891	/* Filter out anything that depends on CPUID levels we don't have */
1892	filter_cpuid_features(c, true);
1893
1894	/* If the model name is still unset, do table lookup. */
1895	if (!c->x86_model_id[0]) {
1896		const char *p;
1897		p = table_lookup_model(c);
1898		if (p)
1899			strcpy(c->x86_model_id, p);
1900		else
1901			/* Last resort... */
1902			sprintf(c->x86_model_id, "%02x/%02x",
1903				c->x86, c->x86_model);
1904	}
1905
1906#ifdef CONFIG_X86_64
1907	detect_ht(c);
1908#endif
1909
1910	x86_init_rdrand(c);
1911	setup_pku(c);
1912	setup_cet(c);
1913
1914	/*
1915	 * Clear/Set all flags overridden by options, need do it
1916	 * before following smp all cpus cap AND.
1917	 */
1918	apply_forced_caps(c);
1919
1920	/*
1921	 * On SMP, boot_cpu_data holds the common feature set between
1922	 * all CPUs; so make sure that we indicate which features are
1923	 * common between the CPUs.  The first time this routine gets
1924	 * executed, c == &boot_cpu_data.
1925	 */
1926	if (c != &boot_cpu_data) {
1927		/* AND the already accumulated flags with these */
1928		for (i = 0; i < NCAPINTS; i++)
1929			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1930
1931		/* OR, i.e. replicate the bug flags */
1932		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1933			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1934	}
1935
1936	ppin_init(c);
1937
1938	/* Init Machine Check Exception if available. */
1939	mcheck_cpu_init(c);
1940
1941	select_idle_routine(c);
1942
1943#ifdef CONFIG_NUMA
1944	numa_add_cpu(smp_processor_id());
1945#endif
1946}
1947
1948/*
1949 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1950 * on 32-bit kernels:
1951 */
1952#ifdef CONFIG_X86_32
1953void enable_sep_cpu(void)
1954{
1955	struct tss_struct *tss;
1956	int cpu;
1957
1958	if (!boot_cpu_has(X86_FEATURE_SEP))
1959		return;
1960
1961	cpu = get_cpu();
1962	tss = &per_cpu(cpu_tss_rw, cpu);
1963
1964	/*
1965	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1966	 * see the big comment in struct x86_hw_tss's definition.
1967	 */
1968
1969	tss->x86_tss.ss1 = __KERNEL_CS;
1970	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1971	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1972	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1973
1974	put_cpu();
1975}
1976#endif
1977
1978static __init void identify_boot_cpu(void)
1979{
1980	identify_cpu(&boot_cpu_data);
1981	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
1982		pr_info("CET detected: Indirect Branch Tracking enabled\n");
1983#ifdef CONFIG_X86_32
1984	enable_sep_cpu();
1985#endif
1986	cpu_detect_tlb(&boot_cpu_data);
1987	setup_cr_pinning();
1988
1989	tsx_init();
1990	tdx_init();
1991	lkgs_init();
1992}
1993
1994void identify_secondary_cpu(struct cpuinfo_x86 *c)
1995{
1996	BUG_ON(c == &boot_cpu_data);
1997	identify_cpu(c);
1998#ifdef CONFIG_X86_32
1999	enable_sep_cpu();
2000#endif
2001	validate_apic_and_package_id(c);
2002	x86_spec_ctrl_setup_ap();
2003	update_srbds_msr();
2004	if (boot_cpu_has_bug(X86_BUG_GDS))
2005		update_gds_msr();
2006
2007	tsx_ap_init();
2008}
2009
2010void print_cpu_info(struct cpuinfo_x86 *c)
2011{
2012	const char *vendor = NULL;
2013
2014	if (c->x86_vendor < X86_VENDOR_NUM) {
2015		vendor = this_cpu->c_vendor;
2016	} else {
2017		if (c->cpuid_level >= 0)
2018			vendor = c->x86_vendor_id;
2019	}
2020
2021	if (vendor && !strstr(c->x86_model_id, vendor))
2022		pr_cont("%s ", vendor);
2023
2024	if (c->x86_model_id[0])
2025		pr_cont("%s", c->x86_model_id);
2026	else
2027		pr_cont("%d86", c->x86);
2028
2029	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
2030
2031	if (c->x86_stepping || c->cpuid_level >= 0)
2032		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
2033	else
2034		pr_cont(")\n");
2035}
2036
2037/*
2038 * clearcpuid= was already parsed in cpu_parse_early_param().  This dummy
2039 * function prevents it from becoming an environment variable for init.
2040 */
2041static __init int setup_clearcpuid(char *arg)
2042{
2043	return 1;
2044}
2045__setup("clearcpuid=", setup_clearcpuid);
2046
2047DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
2048	.current_task	= &init_task,
2049	.preempt_count	= INIT_PREEMPT_COUNT,
2050	.top_of_stack	= TOP_OF_INIT_STACK,
2051};
2052EXPORT_PER_CPU_SYMBOL(pcpu_hot);
 
2053
2054#ifdef CONFIG_X86_64
2055DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
2056		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
2057EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2058
2059static void wrmsrl_cstar(unsigned long val)
2060{
2061	/*
2062	 * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2063	 * is so far ignored by the CPU, but raises a #VE trap in a TDX
2064	 * guest. Avoid the pointless write on all Intel CPUs.
2065	 */
2066	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2067		wrmsrl(MSR_CSTAR, val);
2068}
2069
2070/* May not be marked __init: used by software suspend */
2071void syscall_init(void)
2072{
2073	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2074	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2075
2076	if (ia32_enabled()) {
2077		wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2078		/*
2079		 * This only works on Intel CPUs.
2080		 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2081		 * This does not cause SYSENTER to jump to the wrong location, because
2082		 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2083		 */
2084		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2085		wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2086			    (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2087		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2088	} else {
2089		wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore);
2090		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2091		wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2092		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2093	}
2094
2095	/*
2096	 * Flags to clear on syscall; clear as much as possible
2097	 * to minimize user space-kernel interference.
2098	 */
2099	wrmsrl(MSR_SYSCALL_MASK,
2100	       X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2101	       X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2102	       X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2103	       X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2104	       X86_EFLAGS_AC|X86_EFLAGS_ID);
2105}
2106
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2107#else	/* CONFIG_X86_64 */
2108
2109#ifdef CONFIG_STACKPROTECTOR
2110DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2111EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2112#endif
2113
2114#endif	/* CONFIG_X86_64 */
2115
2116/*
2117 * Clear all 6 debug registers:
2118 */
2119static void clear_all_debug_regs(void)
2120{
2121	int i;
2122
2123	for (i = 0; i < 8; i++) {
2124		/* Ignore db4, db5 */
2125		if ((i == 4) || (i == 5))
2126			continue;
2127
2128		set_debugreg(0, i);
2129	}
2130}
2131
2132#ifdef CONFIG_KGDB
2133/*
2134 * Restore debug regs if using kgdbwait and you have a kernel debugger
2135 * connection established.
2136 */
2137static void dbg_restore_debug_regs(void)
2138{
2139	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2140		arch_kgdb_ops.correct_hw_break();
2141}
2142#else /* ! CONFIG_KGDB */
2143#define dbg_restore_debug_regs()
2144#endif /* ! CONFIG_KGDB */
2145
2146static inline void setup_getcpu(int cpu)
2147{
2148	unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2149	struct desc_struct d = { };
2150
2151	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2152		wrmsr(MSR_TSC_AUX, cpudata, 0);
2153
2154	/* Store CPU and node number in limit. */
2155	d.limit0 = cpudata;
2156	d.limit1 = cpudata >> 16;
2157
2158	d.type = 5;		/* RO data, expand down, accessed */
2159	d.dpl = 3;		/* Visible to user code */
2160	d.s = 1;		/* Not a system segment */
2161	d.p = 1;		/* Present */
2162	d.d = 1;		/* 32-bit */
2163
2164	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2165}
2166
2167#ifdef CONFIG_X86_64
2168static inline void tss_setup_ist(struct tss_struct *tss)
2169{
2170	/* Set up the per-CPU TSS IST stacks */
2171	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2172	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2173	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2174	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2175	/* Only mapped when SEV-ES is active */
2176	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2177}
2178#else /* CONFIG_X86_64 */
2179static inline void tss_setup_ist(struct tss_struct *tss) { }
2180#endif /* !CONFIG_X86_64 */
2181
2182static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2183{
2184	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2185
2186#ifdef CONFIG_X86_IOPL_IOPERM
2187	tss->io_bitmap.prev_max = 0;
2188	tss->io_bitmap.prev_sequence = 0;
2189	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2190	/*
2191	 * Invalidate the extra array entry past the end of the all
2192	 * permission bitmap as required by the hardware.
2193	 */
2194	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2195#endif
2196}
2197
2198/*
2199 * Setup everything needed to handle exceptions from the IDT, including the IST
2200 * exceptions which use paranoid_entry().
2201 */
2202void cpu_init_exception_handling(void)
2203{
2204	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2205	int cpu = raw_smp_processor_id();
2206
2207	/* paranoid_entry() gets the CPU number from the GDT */
2208	setup_getcpu(cpu);
2209
2210	/* IST vectors need TSS to be set up. */
2211	tss_setup_ist(tss);
 
2212	tss_setup_io_bitmap(tss);
2213	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2214
2215	load_TR_desc();
2216
2217	/* GHCB needs to be setup to handle #VC. */
2218	setup_ghcb();
2219
2220	/* Finally load the IDT */
2221	load_current_idt();
 
 
2222}
2223
2224/*
2225 * cpu_init() initializes state that is per-CPU. Some data is already
2226 * initialized (naturally) in the bootstrap process, such as the GDT.  We
2227 * reload it nevertheless, this function acts as a 'CPU state barrier',
2228 * nothing should get across.
2229 */
2230void cpu_init(void)
2231{
2232	struct task_struct *cur = current;
2233	int cpu = raw_smp_processor_id();
2234
2235#ifdef CONFIG_NUMA
2236	if (this_cpu_read(numa_node) == 0 &&
2237	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
2238		set_numa_node(early_cpu_to_node(cpu));
2239#endif
2240	pr_debug("Initializing CPU#%d\n", cpu);
2241
2242	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2243	    boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2244		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2245
2246	if (IS_ENABLED(CONFIG_X86_64)) {
2247		loadsegment(fs, 0);
2248		memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2249		syscall_init();
2250
2251		wrmsrl(MSR_FS_BASE, 0);
2252		wrmsrl(MSR_KERNEL_GS_BASE, 0);
2253		barrier();
2254
2255		x2apic_setup();
2256	}
2257
2258	mmgrab(&init_mm);
2259	cur->active_mm = &init_mm;
2260	BUG_ON(cur->mm);
2261	initialize_tlbstate_and_flush();
2262	enter_lazy_tlb(&init_mm, cur);
2263
2264	/*
2265	 * sp0 points to the entry trampoline stack regardless of what task
2266	 * is running.
2267	 */
2268	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2269
2270	load_mm_ldt(&init_mm);
2271
2272	clear_all_debug_regs();
2273	dbg_restore_debug_regs();
2274
2275	doublefault_init_cpu_tss();
2276
2277	if (is_uv_system())
2278		uv_cpu_init();
2279
2280	load_fixmap_gdt(cpu);
2281}
2282
2283#ifdef CONFIG_MICROCODE_LATE_LOADING
2284/**
2285 * store_cpu_caps() - Store a snapshot of CPU capabilities
2286 * @curr_info: Pointer where to store it
2287 *
2288 * Returns: None
2289 */
2290void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2291{
2292	/* Reload CPUID max function as it might've changed. */
2293	curr_info->cpuid_level = cpuid_eax(0);
2294
2295	/* Copy all capability leafs and pick up the synthetic ones. */
2296	memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2297	       sizeof(curr_info->x86_capability));
2298
2299	/* Get the hardware CPUID leafs */
2300	get_cpu_cap(curr_info);
2301}
2302
2303/**
2304 * microcode_check() - Check if any CPU capabilities changed after an update.
2305 * @prev_info:	CPU capabilities stored before an update.
2306 *
2307 * The microcode loader calls this upon late microcode load to recheck features,
2308 * only when microcode has been updated. Caller holds and CPU hotplug lock.
2309 *
2310 * Return: None
2311 */
2312void microcode_check(struct cpuinfo_x86 *prev_info)
2313{
2314	struct cpuinfo_x86 curr_info;
2315
2316	perf_check_microcode();
2317
2318	amd_check_microcode();
2319
2320	store_cpu_caps(&curr_info);
2321
2322	if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2323		    sizeof(prev_info->x86_capability)))
2324		return;
2325
2326	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2327	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2328}
2329#endif
2330
2331/*
2332 * Invoked from core CPU hotplug code after hotplug operations
2333 */
2334void arch_smt_update(void)
2335{
2336	/* Handle the speculative execution misfeatures */
2337	cpu_bugs_smt_update();
2338	/* Check whether IPI broadcasting can be enabled */
2339	apic_smt_update();
2340}
2341
2342void __init arch_cpu_finalize_init(void)
2343{
 
 
2344	identify_boot_cpu();
2345
 
 
2346	/*
2347	 * identify_boot_cpu() initialized SMT support information, let the
2348	 * core code know.
2349	 */
2350	cpu_smt_set_num_threads(smp_num_siblings, smp_num_siblings);
2351
2352	if (!IS_ENABLED(CONFIG_SMP)) {
2353		pr_info("CPU: ");
2354		print_cpu_info(&boot_cpu_data);
2355	}
2356
2357	cpu_select_mitigations();
2358
2359	arch_smt_update();
2360
2361	if (IS_ENABLED(CONFIG_X86_32)) {
2362		/*
2363		 * Check whether this is a real i386 which is not longer
2364		 * supported and fixup the utsname.
2365		 */
2366		if (boot_cpu_data.x86 < 4)
2367			panic("Kernel requires i486+ for 'invlpg' and other features");
2368
2369		init_utsname()->machine[1] =
2370			'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2371	}
2372
2373	/*
2374	 * Must be before alternatives because it might set or clear
2375	 * feature bits.
2376	 */
2377	fpu__init_system();
2378	fpu__init_cpu();
 
 
 
 
 
 
 
2379
2380	alternative_instructions();
2381
2382	if (IS_ENABLED(CONFIG_X86_64)) {
2383		/*
2384		 * Make sure the first 2MB area is not mapped by huge pages
2385		 * There are typically fixed size MTRRs in there and overlapping
2386		 * MTRRs into large pages causes slow downs.
2387		 *
2388		 * Right now we don't do that with gbpages because there seems
2389		 * very little benefit for that case.
2390		 */
2391		if (!direct_gbpages)
2392			set_memory_4k((unsigned long)__va(0), 1);
2393	} else {
2394		fpu__init_check_bugs();
2395	}
2396
2397	/*
2398	 * This needs to be called before any devices perform DMA
2399	 * operations that might use the SWIOTLB bounce buffers. It will
2400	 * mark the bounce buffers as decrypted so that their usage will
2401	 * not cause "plain-text" data to be decrypted when accessed. It
2402	 * must be called after late_time_init() so that Hyper-V x86/x64
2403	 * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2404	 */
2405	mem_encrypt_init();
2406}