1// SPDX-License-Identifier: GPL-2.0-only
   2/* cpu_feature_enabled() cannot be used this early */
   3#define USE_EARLY_PGTABLE_L5
   4
   5#include <linux/memblock.h>
   6#include <linux/linkage.h>
   7#include <linux/bitops.h>
   8#include <linux/kernel.h>
   9#include <linux/export.h>
  10#include <linux/percpu.h>
  11#include <linux/string.h>
  12#include <linux/ctype.h>
  13#include <linux/delay.h>
  14#include <linux/sched/mm.h>
  15#include <linux/sched/clock.h>
  16#include <linux/sched/task.h>
  17#include <linux/sched/smt.h>
  18#include <linux/init.h>
  19#include <linux/kprobes.h>
  20#include <linux/kgdb.h>
  21#include <linux/mem_encrypt.h>
  22#include <linux/smp.h>
  23#include <linux/cpu.h>
  24#include <linux/io.h>
  25#include <linux/syscore_ops.h>
  26#include <linux/pgtable.h>
  27#include <linux/stackprotector.h>
  28#include <linux/utsname.h>
  29
  30#include <asm/alternative.h>
  31#include <asm/cmdline.h>
  32#include <asm/perf_event.h>
  33#include <asm/mmu_context.h>
  34#include <asm/doublefault.h>
  35#include <asm/archrandom.h>
  36#include <asm/hypervisor.h>
  37#include <asm/processor.h>
  38#include <asm/tlbflush.h>
  39#include <asm/debugreg.h>
  40#include <asm/sections.h>
  41#include <asm/vsyscall.h>
  42#include <linux/topology.h>
  43#include <linux/cpumask.h>
  44#include <linux/atomic.h>
  45#include <asm/proto.h>
  46#include <asm/setup.h>
  47#include <asm/apic.h>
  48#include <asm/desc.h>
  49#include <asm/fpu/api.h>
  50#include <asm/mtrr.h>
  51#include <asm/hwcap2.h>
  52#include <linux/numa.h>
  53#include <asm/numa.h>
  54#include <asm/asm.h>
  55#include <asm/bugs.h>
  56#include <asm/cpu.h>
  57#include <asm/mce.h>
  58#include <asm/msr.h>
  59#include <asm/cacheinfo.h>
  60#include <asm/memtype.h>
  61#include <asm/microcode.h>
 
  62#include <asm/intel-family.h>
  63#include <asm/cpu_device_id.h>
  64#include <asm/fred.h>
  65#include <asm/uv/uv.h>
  66#include <asm/ia32.h>
  67#include <asm/set_memory.h>
  68#include <asm/traps.h>
  69#include <asm/sev.h>
  70#include <asm/tdx.h>
  71
  72#include "cpu.h"
  73
  74DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
  75EXPORT_PER_CPU_SYMBOL(cpu_info);
  76
  77u32 elf_hwcap2 __read_mostly;
  78
  79/* Number of siblings per CPU package */
  80unsigned int __max_threads_per_core __ro_after_init = 1;
  81EXPORT_SYMBOL(__max_threads_per_core);
 
  82
  83unsigned int __max_dies_per_package __ro_after_init = 1;
  84EXPORT_SYMBOL(__max_dies_per_package);
  85
  86unsigned int __max_logical_packages __ro_after_init = 1;
  87EXPORT_SYMBOL(__max_logical_packages);
 
  88
  89unsigned int __num_cores_per_package __ro_after_init = 1;
  90EXPORT_SYMBOL(__num_cores_per_package);
  91
  92unsigned int __num_threads_per_package __ro_after_init = 1;
  93EXPORT_SYMBOL(__num_threads_per_package);
 
 
 
 
 
 
  94
  95static struct ppin_info {
  96	int	feature;
  97	int	msr_ppin_ctl;
  98	int	msr_ppin;
  99} ppin_info[] = {
 100	[X86_VENDOR_INTEL] = {
 101		.feature = X86_FEATURE_INTEL_PPIN,
 102		.msr_ppin_ctl = MSR_PPIN_CTL,
 103		.msr_ppin = MSR_PPIN
 104	},
 105	[X86_VENDOR_AMD] = {
 106		.feature = X86_FEATURE_AMD_PPIN,
 107		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
 108		.msr_ppin = MSR_AMD_PPIN
 109	},
 110};
 111
 112static const struct x86_cpu_id ppin_cpuids[] = {
 113	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
 114	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
 115
 116	/* Legacy models without CPUID enumeration */
 117	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
 118	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 119	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
 120	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
 121	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 122	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
 123	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
 124	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 125	X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
 126	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
 127	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
 128
 129	{}
 130};
 131
 132static void ppin_init(struct cpuinfo_x86 *c)
 133{
 134	const struct x86_cpu_id *id;
 135	unsigned long long val;
 136	struct ppin_info *info;
 137
 138	id = x86_match_cpu(ppin_cpuids);
 139	if (!id)
 140		return;
 141
 142	/*
 143	 * Testing the presence of the MSR is not enough. Need to check
 144	 * that the PPIN_CTL allows reading of the PPIN.
 145	 */
 146	info = (struct ppin_info *)id->driver_data;
 147
 148	if (rdmsrl_safe(info->msr_ppin_ctl, &val))
 149		goto clear_ppin;
 150
 151	if ((val & 3UL) == 1UL) {
 152		/* PPIN locked in disabled mode */
 153		goto clear_ppin;
 154	}
 155
 156	/* If PPIN is disabled, try to enable */
 157	if (!(val & 2UL)) {
 158		wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
 159		rdmsrl_safe(info->msr_ppin_ctl, &val);
 160	}
 161
 162	/* Is the enable bit set? */
 163	if (val & 2UL) {
 164		c->ppin = __rdmsr(info->msr_ppin);
 165		set_cpu_cap(c, info->feature);
 166		return;
 167	}
 168
 169clear_ppin:
 170	clear_cpu_cap(c, info->feature);
 171}
 172
 
 
 
 
 
 
 
 
 
 173static void default_init(struct cpuinfo_x86 *c)
 174{
 175#ifdef CONFIG_X86_64
 176	cpu_detect_cache_sizes(c);
 177#else
 178	/* Not much we can do here... */
 179	/* Check if at least it has cpuid */
 180	if (c->cpuid_level == -1) {
 181		/* No cpuid. It must be an ancient CPU */
 182		if (c->x86 == 4)
 183			strcpy(c->x86_model_id, "486");
 184		else if (c->x86 == 3)
 185			strcpy(c->x86_model_id, "386");
 186	}
 187#endif
 188}
 189
 190static const struct cpu_dev default_cpu = {
 191	.c_init		= default_init,
 192	.c_vendor	= "Unknown",
 193	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
 194};
 195
 196static const struct cpu_dev *this_cpu = &default_cpu;
 197
 198DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 199#ifdef CONFIG_X86_64
 200	/*
 201	 * We need valid kernel segments for data and code in long mode too
 202	 * IRET will check the segment types  kkeil 2000/10/28
 203	 * Also sysret mandates a special GDT layout
 204	 *
 205	 * TLS descriptors are currently at a different place compared to i386.
 206	 * Hopefully nobody expects them at a fixed place (Wine?)
 207	 */
 208	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 209	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
 210	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
 211	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 212	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
 213	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
 214#else
 215	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
 216	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 217	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
 218	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
 219	/*
 220	 * Segments used for calling PnP BIOS have byte granularity.
 221	 * They code segments and data segments have fixed 64k limits,
 222	 * the transfer segment sizes are set at run time.
 223	 */
 224	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 225	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 226	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
 227	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 228	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
 
 
 
 
 
 229	/*
 230	 * The APM segments have byte granularity and their bases
 231	 * are set at run time.  All have 64k limits.
 232	 */
 233	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
 234	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
 235	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
 
 
 
 236
 237	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 238	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
 239#endif
 240} };
 241EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
 242
 243#ifdef CONFIG_X86_64
 244static int __init x86_nopcid_setup(char *s)
 245{
 246	/* nopcid doesn't accept parameters */
 247	if (s)
 248		return -EINVAL;
 249
 250	/* do not emit a message if the feature is not present */
 251	if (!boot_cpu_has(X86_FEATURE_PCID))
 252		return 0;
 253
 254	setup_clear_cpu_cap(X86_FEATURE_PCID);
 255	pr_info("nopcid: PCID feature disabled\n");
 256	return 0;
 257}
 258early_param("nopcid", x86_nopcid_setup);
 259#endif
 260
 261static int __init x86_noinvpcid_setup(char *s)
 262{
 263	/* noinvpcid doesn't accept parameters */
 264	if (s)
 265		return -EINVAL;
 266
 267	/* do not emit a message if the feature is not present */
 268	if (!boot_cpu_has(X86_FEATURE_INVPCID))
 269		return 0;
 270
 271	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
 272	pr_info("noinvpcid: INVPCID feature disabled\n");
 273	return 0;
 274}
 275early_param("noinvpcid", x86_noinvpcid_setup);
 276
 277#ifdef CONFIG_X86_32
 278static int cachesize_override = -1;
 279static int disable_x86_serial_nr = 1;
 280
 281static int __init cachesize_setup(char *str)
 282{
 283	get_option(&str, &cachesize_override);
 284	return 1;
 285}
 286__setup("cachesize=", cachesize_setup);
 287
 288/* Standard macro to see if a specific flag is changeable */
 289static inline int flag_is_changeable_p(u32 flag)
 290{
 291	u32 f1, f2;
 292
 293	/*
 294	 * Cyrix and IDT cpus allow disabling of CPUID
 295	 * so the code below may return different results
 296	 * when it is executed before and after enabling
 297	 * the CPUID. Add "volatile" to not allow gcc to
 298	 * optimize the subsequent calls to this function.
 299	 */
 300	asm volatile ("pushfl		\n\t"
 301		      "pushfl		\n\t"
 302		      "popl %0		\n\t"
 303		      "movl %0, %1	\n\t"
 304		      "xorl %2, %0	\n\t"
 305		      "pushl %0		\n\t"
 306		      "popfl		\n\t"
 307		      "pushfl		\n\t"
 308		      "popl %0		\n\t"
 309		      "popfl		\n\t"
 310
 311		      : "=&r" (f1), "=&r" (f2)
 312		      : "ir" (flag));
 313
 314	return ((f1^f2) & flag) != 0;
 315}
 316
 317/* Probe for the CPUID instruction */
 318int have_cpuid_p(void)
 319{
 320	return flag_is_changeable_p(X86_EFLAGS_ID);
 321}
 322
 323static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 324{
 325	unsigned long lo, hi;
 326
 327	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
 328		return;
 329
 330	/* Disable processor serial number: */
 331
 332	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 333	lo |= 0x200000;
 334	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
 335
 336	pr_notice("CPU serial number disabled.\n");
 337	clear_cpu_cap(c, X86_FEATURE_PN);
 338
 339	/* Disabling the serial number may affect the cpuid level */
 340	c->cpuid_level = cpuid_eax(0);
 341}
 342
 343static int __init x86_serial_nr_setup(char *s)
 344{
 345	disable_x86_serial_nr = 0;
 346	return 1;
 347}
 348__setup("serialnumber", x86_serial_nr_setup);
 349#else
 350static inline int flag_is_changeable_p(u32 flag)
 351{
 352	return 1;
 353}
 354static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 355{
 356}
 357#endif
 358
 359static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 360{
 361	if (cpu_has(c, X86_FEATURE_SMEP))
 362		cr4_set_bits(X86_CR4_SMEP);
 363}
 364
 365static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 366{
 367	unsigned long eflags = native_save_fl();
 368
 369	/* This should have been cleared long ago */
 370	BUG_ON(eflags & X86_EFLAGS_AC);
 371
 372	if (cpu_has(c, X86_FEATURE_SMAP))
 373		cr4_set_bits(X86_CR4_SMAP);
 374}
 375
 376static __always_inline void setup_umip(struct cpuinfo_x86 *c)
 377{
 378	/* Check the boot processor, plus build option for UMIP. */
 379	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
 380		goto out;
 381
 382	/* Check the current processor's cpuid bits. */
 383	if (!cpu_has(c, X86_FEATURE_UMIP))
 384		goto out;
 385
 386	cr4_set_bits(X86_CR4_UMIP);
 387
 388	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
 389
 390	return;
 391
 392out:
 393	/*
 394	 * Make sure UMIP is disabled in case it was enabled in a
 395	 * previous boot (e.g., via kexec).
 396	 */
 397	cr4_clear_bits(X86_CR4_UMIP);
 398}
 399
 400/* These bits should not change their value after CPU init is finished. */
 401static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
 402					     X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED;
 
 403static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
 404static unsigned long cr4_pinned_bits __ro_after_init;
 405
 406void native_write_cr0(unsigned long val)
 407{
 408	unsigned long bits_missing = 0;
 409
 410set_register:
 411	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
 412
 413	if (static_branch_likely(&cr_pinning)) {
 414		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
 415			bits_missing = X86_CR0_WP;
 416			val |= bits_missing;
 417			goto set_register;
 418		}
 419		/* Warn after we've set the missing bits. */
 420		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
 421	}
 422}
 423EXPORT_SYMBOL(native_write_cr0);
 424
 425void __no_profile native_write_cr4(unsigned long val)
 426{
 427	unsigned long bits_changed = 0;
 428
 429set_register:
 430	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
 431
 432	if (static_branch_likely(&cr_pinning)) {
 433		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
 434			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
 435			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
 436			goto set_register;
 437		}
 438		/* Warn after we've corrected the changed bits. */
 439		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
 440			  bits_changed);
 441	}
 442}
 443#if IS_MODULE(CONFIG_LKDTM)
 444EXPORT_SYMBOL_GPL(native_write_cr4);
 445#endif
 446
 447void cr4_update_irqsoff(unsigned long set, unsigned long clear)
 448{
 449	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
 450
 451	lockdep_assert_irqs_disabled();
 452
 453	newval = (cr4 & ~clear) | set;
 454	if (newval != cr4) {
 455		this_cpu_write(cpu_tlbstate.cr4, newval);
 456		__write_cr4(newval);
 457	}
 458}
 459EXPORT_SYMBOL(cr4_update_irqsoff);
 460
 461/* Read the CR4 shadow. */
 462unsigned long cr4_read_shadow(void)
 463{
 464	return this_cpu_read(cpu_tlbstate.cr4);
 465}
 466EXPORT_SYMBOL_GPL(cr4_read_shadow);
 467
 468void cr4_init(void)
 469{
 470	unsigned long cr4 = __read_cr4();
 471
 472	if (boot_cpu_has(X86_FEATURE_PCID))
 473		cr4 |= X86_CR4_PCIDE;
 474	if (static_branch_likely(&cr_pinning))
 475		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
 476
 477	__write_cr4(cr4);
 478
 479	/* Initialize cr4 shadow for this CPU. */
 480	this_cpu_write(cpu_tlbstate.cr4, cr4);
 481}
 482
 483/*
 484 * Once CPU feature detection is finished (and boot params have been
 485 * parsed), record any of the sensitive CR bits that are set, and
 486 * enable CR pinning.
 487 */
 488static void __init setup_cr_pinning(void)
 489{
 490	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
 491	static_key_enable(&cr_pinning.key);
 492}
 493
 494static __init int x86_nofsgsbase_setup(char *arg)
 495{
 496	/* Require an exact match without trailing characters. */
 497	if (strlen(arg))
 498		return 0;
 499
 500	/* Do not emit a message if the feature is not present. */
 501	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
 502		return 1;
 503
 504	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
 505	pr_info("FSGSBASE disabled via kernel command line\n");
 506	return 1;
 507}
 508__setup("nofsgsbase", x86_nofsgsbase_setup);
 509
 510/*
 511 * Protection Keys are not available in 32-bit mode.
 512 */
 513static bool pku_disabled;
 514
 515static __always_inline void setup_pku(struct cpuinfo_x86 *c)
 516{
 517	if (c == &boot_cpu_data) {
 518		if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
 519			return;
 520		/*
 521		 * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
 522		 * bit to be set.  Enforce it.
 523		 */
 524		setup_force_cpu_cap(X86_FEATURE_OSPKE);
 525
 526	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
 527		return;
 528	}
 529
 530	cr4_set_bits(X86_CR4_PKE);
 531	/* Load the default PKRU value */
 532	pkru_write_default();
 533}
 534
 535#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
 536static __init int setup_disable_pku(char *arg)
 537{
 538	/*
 539	 * Do not clear the X86_FEATURE_PKU bit.  All of the
 540	 * runtime checks are against OSPKE so clearing the
 541	 * bit does nothing.
 542	 *
 543	 * This way, we will see "pku" in cpuinfo, but not
 544	 * "ospke", which is exactly what we want.  It shows
 545	 * that the CPU has PKU, but the OS has not enabled it.
 546	 * This happens to be exactly how a system would look
 547	 * if we disabled the config option.
 548	 */
 549	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
 550	pku_disabled = true;
 551	return 1;
 552}
 553__setup("nopku", setup_disable_pku);
 554#endif
 555
 556#ifdef CONFIG_X86_KERNEL_IBT
 557
 558__noendbr u64 ibt_save(bool disable)
 559{
 560	u64 msr = 0;
 561
 562	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 563		rdmsrl(MSR_IA32_S_CET, msr);
 564		if (disable)
 565			wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
 566	}
 567
 568	return msr;
 569}
 570
 571__noendbr void ibt_restore(u64 save)
 572{
 573	u64 msr;
 574
 575	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
 576		rdmsrl(MSR_IA32_S_CET, msr);
 577		msr &= ~CET_ENDBR_EN;
 578		msr |= (save & CET_ENDBR_EN);
 579		wrmsrl(MSR_IA32_S_CET, msr);
 580	}
 581}
 582
 583#endif
 584
 585static __always_inline void setup_cet(struct cpuinfo_x86 *c)
 586{
 587	bool user_shstk, kernel_ibt;
 588
 589	if (!IS_ENABLED(CONFIG_X86_CET))
 
 590		return;
 591
 592	kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
 593	user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
 594		     IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
 595
 596	if (!kernel_ibt && !user_shstk)
 597		return;
 598
 599	if (user_shstk)
 600		set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
 601
 602	if (kernel_ibt)
 603		wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
 604	else
 605		wrmsrl(MSR_IA32_S_CET, 0);
 606
 607	cr4_set_bits(X86_CR4_CET);
 608
 609	if (kernel_ibt && ibt_selftest()) {
 610		pr_err("IBT selftest: Failed!\n");
 611		wrmsrl(MSR_IA32_S_CET, 0);
 612		setup_clear_cpu_cap(X86_FEATURE_IBT);
 
 613	}
 614}
 615
 616__noendbr void cet_disable(void)
 617{
 618	if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
 619	      cpu_feature_enabled(X86_FEATURE_SHSTK)))
 620		return;
 621
 622	wrmsrl(MSR_IA32_S_CET, 0);
 623	wrmsrl(MSR_IA32_U_CET, 0);
 624}
 625
 626/*
 627 * Some CPU features depend on higher CPUID levels, which may not always
 628 * be available due to CPUID level capping or broken virtualization
 629 * software.  Add those features to this table to auto-disable them.
 630 */
 631struct cpuid_dependent_feature {
 632	u32 feature;
 633	u32 level;
 634};
 635
 636static const struct cpuid_dependent_feature
 637cpuid_dependent_features[] = {
 638	{ X86_FEATURE_MWAIT,		0x00000005 },
 639	{ X86_FEATURE_DCA,		0x00000009 },
 640	{ X86_FEATURE_XSAVE,		0x0000000d },
 641	{ 0, 0 }
 642};
 643
 644static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
 645{
 646	const struct cpuid_dependent_feature *df;
 647
 648	for (df = cpuid_dependent_features; df->feature; df++) {
 649
 650		if (!cpu_has(c, df->feature))
 651			continue;
 652		/*
 653		 * Note: cpuid_level is set to -1 if unavailable, but
 654		 * extended_extended_level is set to 0 if unavailable
 655		 * and the legitimate extended levels are all negative
 656		 * when signed; hence the weird messing around with
 657		 * signs here...
 658		 */
 659		if (!((s32)df->level < 0 ?
 660		     (u32)df->level > (u32)c->extended_cpuid_level :
 661		     (s32)df->level > (s32)c->cpuid_level))
 662			continue;
 663
 664		clear_cpu_cap(c, df->feature);
 665		if (!warn)
 666			continue;
 667
 668		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
 669			x86_cap_flag(df->feature), df->level);
 670	}
 671}
 672
 673/*
 674 * Naming convention should be: <Name> [(<Codename>)]
 675 * This table only is used unless init_<vendor>() below doesn't set it;
 676 * in particular, if CPUID levels 0x80000002..4 are supported, this
 677 * isn't used
 678 */
 679
 680/* Look up CPU names by table lookup. */
 681static const char *table_lookup_model(struct cpuinfo_x86 *c)
 682{
 683#ifdef CONFIG_X86_32
 684	const struct legacy_cpu_model_info *info;
 685
 686	if (c->x86_model >= 16)
 687		return NULL;	/* Range check */
 688
 689	if (!this_cpu)
 690		return NULL;
 691
 692	info = this_cpu->legacy_models;
 693
 694	while (info->family) {
 695		if (info->family == c->x86)
 696			return info->model_names[c->x86_model];
 697		info++;
 698	}
 699#endif
 700	return NULL;		/* Not found */
 701}
 702
 703/* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
 704__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 705__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
 706
 707#ifdef CONFIG_X86_32
 708/* The 32-bit entry code needs to find cpu_entry_area. */
 709DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 710#endif
 711
 712/* Load the original GDT from the per-cpu structure */
 713void load_direct_gdt(int cpu)
 714{
 715	struct desc_ptr gdt_descr;
 716
 717	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
 718	gdt_descr.size = GDT_SIZE - 1;
 719	load_gdt(&gdt_descr);
 720}
 721EXPORT_SYMBOL_GPL(load_direct_gdt);
 722
 723/* Load a fixmap remapping of the per-cpu GDT */
 724void load_fixmap_gdt(int cpu)
 725{
 726	struct desc_ptr gdt_descr;
 727
 728	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
 729	gdt_descr.size = GDT_SIZE - 1;
 730	load_gdt(&gdt_descr);
 731}
 732EXPORT_SYMBOL_GPL(load_fixmap_gdt);
 733
 734/**
 735 * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
 736 * @cpu:	The CPU number for which this is invoked
 737 *
 738 * Invoked during early boot to switch from early GDT and early per CPU to
 739 * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
 740 * switch is implicit by loading the direct GDT. On 64bit this requires
 741 * to update GSBASE.
 742 */
 743void __init switch_gdt_and_percpu_base(int cpu)
 744{
 745	load_direct_gdt(cpu);
 746
 747#ifdef CONFIG_X86_64
 748	/*
 749	 * No need to load %gs. It is already correct.
 750	 *
 751	 * Writing %gs on 64bit would zero GSBASE which would make any per
 752	 * CPU operation up to the point of the wrmsrl() fault.
 753	 *
 754	 * Set GSBASE to the new offset. Until the wrmsrl() happens the
 755	 * early mapping is still valid. That means the GSBASE update will
 756	 * lose any prior per CPU data which was not copied over in
 757	 * setup_per_cpu_areas().
 758	 *
 759	 * This works even with stackprotector enabled because the
 760	 * per CPU stack canary is 0 in both per CPU areas.
 761	 */
 762	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
 763#else
 764	/*
 765	 * %fs is already set to __KERNEL_PERCPU, but after switching GDT
 766	 * it is required to load FS again so that the 'hidden' part is
 767	 * updated from the new GDT. Up to this point the early per CPU
 768	 * translation is active. Any content of the early per CPU data
 769	 * which was not copied over in setup_per_cpu_areas() is lost.
 770	 */
 771	loadsegment(fs, __KERNEL_PERCPU);
 772#endif
 773}
 774
 775static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
 776
 777static void get_model_name(struct cpuinfo_x86 *c)
 778{
 779	unsigned int *v;
 780	char *p, *q, *s;
 781
 782	if (c->extended_cpuid_level < 0x80000004)
 783		return;
 784
 785	v = (unsigned int *)c->x86_model_id;
 786	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
 787	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
 788	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
 789	c->x86_model_id[48] = 0;
 790
 791	/* Trim whitespace */
 792	p = q = s = &c->x86_model_id[0];
 793
 794	while (*p == ' ')
 795		p++;
 796
 797	while (*p) {
 798		/* Note the last non-whitespace index */
 799		if (!isspace(*p))
 800			s = q;
 801
 802		*q++ = *p++;
 803	}
 804
 805	*(s + 1) = '\0';
 806}
 807
 
 
 
 
 
 
 
 
 
 
 
 
 
 808void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 809{
 810	unsigned int n, dummy, ebx, ecx, edx, l2size;
 811
 812	n = c->extended_cpuid_level;
 813
 814	if (n >= 0x80000005) {
 815		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
 816		c->x86_cache_size = (ecx>>24) + (edx>>24);
 817#ifdef CONFIG_X86_64
 818		/* On K8 L1 TLB is inclusive, so don't count it */
 819		c->x86_tlbsize = 0;
 820#endif
 821	}
 822
 823	if (n < 0x80000006)	/* Some chips just has a large L1. */
 824		return;
 825
 826	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
 827	l2size = ecx >> 16;
 828
 829#ifdef CONFIG_X86_64
 830	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
 831#else
 832	/* do processor-specific cache resizing */
 833	if (this_cpu->legacy_cache_size)
 834		l2size = this_cpu->legacy_cache_size(c, l2size);
 835
 836	/* Allow user to override all this if necessary. */
 837	if (cachesize_override != -1)
 838		l2size = cachesize_override;
 839
 840	if (l2size == 0)
 841		return;		/* Again, no L2 cache is possible */
 842#endif
 843
 844	c->x86_cache_size = l2size;
 845}
 846
 847u16 __read_mostly tlb_lli_4k[NR_INFO];
 848u16 __read_mostly tlb_lli_2m[NR_INFO];
 849u16 __read_mostly tlb_lli_4m[NR_INFO];
 850u16 __read_mostly tlb_lld_4k[NR_INFO];
 851u16 __read_mostly tlb_lld_2m[NR_INFO];
 852u16 __read_mostly tlb_lld_4m[NR_INFO];
 853u16 __read_mostly tlb_lld_1g[NR_INFO];
 854
 855static void cpu_detect_tlb(struct cpuinfo_x86 *c)
 856{
 857	if (this_cpu->c_detect_tlb)
 858		this_cpu->c_detect_tlb(c);
 859
 860	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
 861		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
 862		tlb_lli_4m[ENTRIES]);
 863
 864	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
 865		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
 866		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
 867}
 868
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 869static void get_cpu_vendor(struct cpuinfo_x86 *c)
 870{
 871	char *v = c->x86_vendor_id;
 872	int i;
 873
 874	for (i = 0; i < X86_VENDOR_NUM; i++) {
 875		if (!cpu_devs[i])
 876			break;
 877
 878		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
 879		    (cpu_devs[i]->c_ident[1] &&
 880		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
 881
 882			this_cpu = cpu_devs[i];
 883			c->x86_vendor = this_cpu->c_x86_vendor;
 884			return;
 885		}
 886	}
 887
 888	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
 889		    "CPU: Your system may be unstable.\n", v);
 890
 891	c->x86_vendor = X86_VENDOR_UNKNOWN;
 892	this_cpu = &default_cpu;
 893}
 894
 895void cpu_detect(struct cpuinfo_x86 *c)
 896{
 897	/* Get vendor name */
 898	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
 899	      (unsigned int *)&c->x86_vendor_id[0],
 900	      (unsigned int *)&c->x86_vendor_id[8],
 901	      (unsigned int *)&c->x86_vendor_id[4]);
 902
 903	c->x86 = 4;
 904	/* Intel-defined flags: level 0x00000001 */
 905	if (c->cpuid_level >= 0x00000001) {
 906		u32 junk, tfms, cap0, misc;
 907
 908		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
 909		c->x86		= x86_family(tfms);
 910		c->x86_model	= x86_model(tfms);
 911		c->x86_stepping	= x86_stepping(tfms);
 912
 913		if (cap0 & (1<<19)) {
 914			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
 915			c->x86_cache_alignment = c->x86_clflush_size;
 916		}
 917	}
 918}
 919
 920static void apply_forced_caps(struct cpuinfo_x86 *c)
 921{
 922	int i;
 923
 924	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
 925		c->x86_capability[i] &= ~cpu_caps_cleared[i];
 926		c->x86_capability[i] |= cpu_caps_set[i];
 927	}
 928}
 929
 930static void init_speculation_control(struct cpuinfo_x86 *c)
 931{
 932	/*
 933	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
 934	 * and they also have a different bit for STIBP support. Also,
 935	 * a hypervisor might have set the individual AMD bits even on
 936	 * Intel CPUs, for finer-grained selection of what's available.
 937	 */
 938	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
 939		set_cpu_cap(c, X86_FEATURE_IBRS);
 940		set_cpu_cap(c, X86_FEATURE_IBPB);
 941		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 942	}
 943
 944	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
 945		set_cpu_cap(c, X86_FEATURE_STIBP);
 946
 947	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
 948	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
 949		set_cpu_cap(c, X86_FEATURE_SSBD);
 950
 951	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
 952		set_cpu_cap(c, X86_FEATURE_IBRS);
 953		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 954	}
 955
 956	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
 957		set_cpu_cap(c, X86_FEATURE_IBPB);
 958
 959	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
 960		set_cpu_cap(c, X86_FEATURE_STIBP);
 961		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 962	}
 963
 964	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
 965		set_cpu_cap(c, X86_FEATURE_SSBD);
 966		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
 967		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
 968	}
 969}
 970
 971void get_cpu_cap(struct cpuinfo_x86 *c)
 972{
 973	u32 eax, ebx, ecx, edx;
 974
 975	/* Intel-defined flags: level 0x00000001 */
 976	if (c->cpuid_level >= 0x00000001) {
 977		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
 978
 979		c->x86_capability[CPUID_1_ECX] = ecx;
 980		c->x86_capability[CPUID_1_EDX] = edx;
 981	}
 982
 983	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
 984	if (c->cpuid_level >= 0x00000006)
 985		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
 986
 987	/* Additional Intel-defined flags: level 0x00000007 */
 988	if (c->cpuid_level >= 0x00000007) {
 989		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
 990		c->x86_capability[CPUID_7_0_EBX] = ebx;
 991		c->x86_capability[CPUID_7_ECX] = ecx;
 992		c->x86_capability[CPUID_7_EDX] = edx;
 993
 994		/* Check valid sub-leaf index before accessing it */
 995		if (eax >= 1) {
 996			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
 997			c->x86_capability[CPUID_7_1_EAX] = eax;
 998		}
 999	}
1000
1001	/* Extended state features: level 0x0000000d */
1002	if (c->cpuid_level >= 0x0000000d) {
1003		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1004
1005		c->x86_capability[CPUID_D_1_EAX] = eax;
1006	}
1007
1008	/* AMD-defined flags: level 0x80000001 */
1009	eax = cpuid_eax(0x80000000);
1010	c->extended_cpuid_level = eax;
1011
1012	if ((eax & 0xffff0000) == 0x80000000) {
1013		if (eax >= 0x80000001) {
1014			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1015
1016			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1017			c->x86_capability[CPUID_8000_0001_EDX] = edx;
1018		}
1019	}
1020
1021	if (c->extended_cpuid_level >= 0x80000007) {
1022		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1023
1024		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1025		c->x86_power = edx;
1026	}
1027
1028	if (c->extended_cpuid_level >= 0x80000008) {
1029		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1030		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1031	}
1032
1033	if (c->extended_cpuid_level >= 0x8000000a)
1034		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1035
1036	if (c->extended_cpuid_level >= 0x8000001f)
1037		c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1038
1039	if (c->extended_cpuid_level >= 0x80000021)
1040		c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
1041
1042	init_scattered_cpuid_features(c);
1043	init_speculation_control(c);
1044
1045	/*
1046	 * Clear/Set all flags overridden by options, after probe.
1047	 * This needs to happen each time we re-probe, which may happen
1048	 * several times during CPU initialization.
1049	 */
1050	apply_forced_caps(c);
1051}
1052
1053void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1054{
1055	u32 eax, ebx, ecx, edx;
1056	bool vp_bits_from_cpuid = true;
1057
1058	if (!cpu_has(c, X86_FEATURE_CPUID) ||
1059	    (c->extended_cpuid_level < 0x80000008))
1060		vp_bits_from_cpuid = false;
1061
1062	if (vp_bits_from_cpuid) {
1063		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1064
1065		c->x86_virt_bits = (eax >> 8) & 0xff;
1066		c->x86_phys_bits = eax & 0xff;
1067	} else {
1068		if (IS_ENABLED(CONFIG_X86_64)) {
1069			c->x86_clflush_size = 64;
1070			c->x86_phys_bits = 36;
1071			c->x86_virt_bits = 48;
1072		} else {
1073			c->x86_clflush_size = 32;
1074			c->x86_virt_bits = 32;
1075			c->x86_phys_bits = 32;
1076
1077			if (cpu_has(c, X86_FEATURE_PAE) ||
1078			    cpu_has(c, X86_FEATURE_PSE36))
1079				c->x86_phys_bits = 36;
1080		}
1081	}
 
 
 
 
1082	c->x86_cache_bits = c->x86_phys_bits;
1083	c->x86_cache_alignment = c->x86_clflush_size;
1084}
1085
1086static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1087{
1088#ifdef CONFIG_X86_32
1089	int i;
1090
1091	/*
1092	 * First of all, decide if this is a 486 or higher
1093	 * It's a 486 if we can modify the AC flag
1094	 */
1095	if (flag_is_changeable_p(X86_EFLAGS_AC))
1096		c->x86 = 4;
1097	else
1098		c->x86 = 3;
1099
1100	for (i = 0; i < X86_VENDOR_NUM; i++)
1101		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1102			c->x86_vendor_id[0] = 0;
1103			cpu_devs[i]->c_identify(c);
1104			if (c->x86_vendor_id[0]) {
1105				get_cpu_vendor(c);
1106				break;
1107			}
1108		}
1109#endif
1110}
1111
1112#define NO_SPECULATION		BIT(0)
1113#define NO_MELTDOWN		BIT(1)
1114#define NO_SSB			BIT(2)
1115#define NO_L1TF			BIT(3)
1116#define NO_MDS			BIT(4)
1117#define MSBDS_ONLY		BIT(5)
1118#define NO_SWAPGS		BIT(6)
1119#define NO_ITLB_MULTIHIT	BIT(7)
1120#define NO_SPECTRE_V2		BIT(8)
1121#define NO_MMIO			BIT(9)
1122#define NO_EIBRS_PBRSB		BIT(10)
1123#define NO_BHI			BIT(11)
1124
1125#define VULNWL(vendor, family, model, whitelist)	\
1126	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1127
1128#define VULNWL_INTEL(model, whitelist)		\
1129	VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1130
1131#define VULNWL_AMD(family, whitelist)		\
1132	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1133
1134#define VULNWL_HYGON(family, whitelist)		\
1135	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1136
1137static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1138	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
1139	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
1140	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
1141	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),
1142	VULNWL(VORTEX,	5, X86_MODEL_ANY,	NO_SPECULATION),
1143	VULNWL(VORTEX,	6, X86_MODEL_ANY,	NO_SPECULATION),
1144
1145	/* Intel Family 6 */
1146	VULNWL_INTEL(TIGERLAKE,			NO_MMIO),
1147	VULNWL_INTEL(TIGERLAKE_L,		NO_MMIO),
1148	VULNWL_INTEL(ALDERLAKE,			NO_MMIO),
1149	VULNWL_INTEL(ALDERLAKE_L,		NO_MMIO),
1150
1151	VULNWL_INTEL(ATOM_SALTWELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1152	VULNWL_INTEL(ATOM_SALTWELL_TABLET,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1153	VULNWL_INTEL(ATOM_SALTWELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1154	VULNWL_INTEL(ATOM_BONNELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1155	VULNWL_INTEL(ATOM_BONNELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1156
1157	VULNWL_INTEL(ATOM_SILVERMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1158	VULNWL_INTEL(ATOM_SILVERMONT_D,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1159	VULNWL_INTEL(ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1160	VULNWL_INTEL(ATOM_AIRMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1161	VULNWL_INTEL(XEON_PHI_KNL,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1162	VULNWL_INTEL(XEON_PHI_KNM,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1163
1164	VULNWL_INTEL(CORE_YONAH,		NO_SSB),
1165
1166	VULNWL_INTEL(ATOM_AIRMONT_MID,		NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1167	VULNWL_INTEL(ATOM_AIRMONT_NP,		NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1168
1169	VULNWL_INTEL(ATOM_GOLDMONT,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1170	VULNWL_INTEL(ATOM_GOLDMONT_D,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1171	VULNWL_INTEL(ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1172
1173	/*
1174	 * Technically, swapgs isn't serializing on AMD (despite it previously
1175	 * being documented as such in the APM).  But according to AMD, %gs is
1176	 * updated non-speculatively, and the issuing of %gs-relative memory
1177	 * operands will be blocked until the %gs update completes, which is
1178	 * good enough for our purposes.
1179	 */
1180
1181	VULNWL_INTEL(ATOM_TREMONT,		NO_EIBRS_PBRSB),
1182	VULNWL_INTEL(ATOM_TREMONT_L,		NO_EIBRS_PBRSB),
1183	VULNWL_INTEL(ATOM_TREMONT_D,		NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1184
1185	/* AMD Family 0xf - 0x12 */
1186	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1187	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1188	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1189	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1190
1191	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1192	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1193	VULNWL_HYGON(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1194
1195	/* Zhaoxin Family 7 */
1196	VULNWL(CENTAUR,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1197	VULNWL(ZHAOXIN,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1198	{}
1199};
1200
1201#define VULNBL(vendor, family, model, blacklist)	\
1202	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1203
1204#define VULNBL_INTEL_STEPPINGS(model, steppings, issues)		   \
1205	X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6,		   \
1206					    INTEL_FAM6_##model, steppings, \
1207					    X86_FEATURE_ANY, issues)
1208
1209#define VULNBL_AMD(family, blacklist)		\
1210	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1211
1212#define VULNBL_HYGON(family, blacklist)		\
1213	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1214
1215#define SRBDS		BIT(0)
1216/* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1217#define MMIO		BIT(1)
1218/* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1219#define MMIO_SBDS	BIT(2)
1220/* CPU is affected by RETbleed, speculating where you would not expect it */
1221#define RETBLEED	BIT(3)
1222/* CPU is affected by SMT (cross-thread) return predictions */
1223#define SMT_RSB		BIT(4)
1224/* CPU is affected by SRSO */
1225#define SRSO		BIT(5)
1226/* CPU is affected by GDS */
1227#define GDS		BIT(6)
1228/* CPU is affected by Register File Data Sampling */
1229#define RFDS		BIT(7)
1230
1231static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1232	VULNBL_INTEL_STEPPINGS(IVYBRIDGE,	X86_STEPPING_ANY,		SRBDS),
1233	VULNBL_INTEL_STEPPINGS(HASWELL,		X86_STEPPING_ANY,		SRBDS),
1234	VULNBL_INTEL_STEPPINGS(HASWELL_L,	X86_STEPPING_ANY,		SRBDS),
1235	VULNBL_INTEL_STEPPINGS(HASWELL_G,	X86_STEPPING_ANY,		SRBDS),
1236	VULNBL_INTEL_STEPPINGS(HASWELL_X,	X86_STEPPING_ANY,		MMIO),
1237	VULNBL_INTEL_STEPPINGS(BROADWELL_D,	X86_STEPPING_ANY,		MMIO),
1238	VULNBL_INTEL_STEPPINGS(BROADWELL_G,	X86_STEPPING_ANY,		SRBDS),
1239	VULNBL_INTEL_STEPPINGS(BROADWELL_X,	X86_STEPPING_ANY,		MMIO),
1240	VULNBL_INTEL_STEPPINGS(BROADWELL,	X86_STEPPING_ANY,		SRBDS),
1241	VULNBL_INTEL_STEPPINGS(SKYLAKE_X,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS),
1242	VULNBL_INTEL_STEPPINGS(SKYLAKE_L,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1243	VULNBL_INTEL_STEPPINGS(SKYLAKE,		X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1244	VULNBL_INTEL_STEPPINGS(KABYLAKE_L,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1245	VULNBL_INTEL_STEPPINGS(KABYLAKE,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS | SRBDS),
1246	VULNBL_INTEL_STEPPINGS(CANNONLAKE_L,	X86_STEPPING_ANY,		RETBLEED),
1247	VULNBL_INTEL_STEPPINGS(ICELAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1248	VULNBL_INTEL_STEPPINGS(ICELAKE_D,	X86_STEPPING_ANY,		MMIO | GDS),
1249	VULNBL_INTEL_STEPPINGS(ICELAKE_X,	X86_STEPPING_ANY,		MMIO | GDS),
1250	VULNBL_INTEL_STEPPINGS(COMETLAKE,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1251	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPINGS(0x0, 0x0),	MMIO | RETBLEED),
1252	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED | GDS),
1253	VULNBL_INTEL_STEPPINGS(TIGERLAKE_L,	X86_STEPPING_ANY,		GDS),
1254	VULNBL_INTEL_STEPPINGS(TIGERLAKE,	X86_STEPPING_ANY,		GDS),
1255	VULNBL_INTEL_STEPPINGS(LAKEFIELD,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1256	VULNBL_INTEL_STEPPINGS(ROCKETLAKE,	X86_STEPPING_ANY,		MMIO | RETBLEED | GDS),
1257	VULNBL_INTEL_STEPPINGS(ALDERLAKE,	X86_STEPPING_ANY,		RFDS),
1258	VULNBL_INTEL_STEPPINGS(ALDERLAKE_L,	X86_STEPPING_ANY,		RFDS),
1259	VULNBL_INTEL_STEPPINGS(RAPTORLAKE,	X86_STEPPING_ANY,		RFDS),
1260	VULNBL_INTEL_STEPPINGS(RAPTORLAKE_P,	X86_STEPPING_ANY,		RFDS),
1261	VULNBL_INTEL_STEPPINGS(RAPTORLAKE_S,	X86_STEPPING_ANY,		RFDS),
1262	VULNBL_INTEL_STEPPINGS(ATOM_GRACEMONT,	X86_STEPPING_ANY,		RFDS),
1263	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RFDS),
1264	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D,	X86_STEPPING_ANY,		MMIO | RFDS),
1265	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RFDS),
1266	VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT,	X86_STEPPING_ANY,		RFDS),
1267	VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_D,	X86_STEPPING_ANY,		RFDS),
1268	VULNBL_INTEL_STEPPINGS(ATOM_GOLDMONT_PLUS, X86_STEPPING_ANY,		RFDS),
1269
1270	VULNBL_AMD(0x15, RETBLEED),
1271	VULNBL_AMD(0x16, RETBLEED),
1272	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1273	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1274	VULNBL_AMD(0x19, SRSO),
1275	{}
1276};
1277
1278static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1279{
1280	const struct x86_cpu_id *m = x86_match_cpu(table);
1281
1282	return m && !!(m->driver_data & which);
1283}
1284
1285u64 x86_read_arch_cap_msr(void)
1286{
1287	u64 x86_arch_cap_msr = 0;
1288
1289	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1290		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
1291
1292	return x86_arch_cap_msr;
1293}
1294
1295static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
1296{
1297	return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
1298		x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
1299		x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
1300}
1301
1302static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
1303{
1304	/* The "immunity" bit trumps everything else: */
1305	if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
1306		return false;
1307
1308	/*
1309	 * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1310	 * indicate that mitigation is needed because guest is running on a
1311	 * vulnerable hardware or may migrate to such hardware:
1312	 */
1313	if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
1314		return true;
1315
1316	/* Only consult the blacklist when there is no enumeration: */
1317	return cpu_matches(cpu_vuln_blacklist, RFDS);
1318}
1319
1320static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1321{
1322	u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
1323
1324	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1325	if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1326	    !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
1327		setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1328
1329	if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1330		return;
1331
1332	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1333
1334	if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1335		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1336
1337	if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1338	    !(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
1339	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1340		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1341
1342	/*
1343	 * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1344	 * flag and protect from vendor-specific bugs via the whitelist.
1345	 *
1346	 * Don't use AutoIBRS when SNP is enabled because it degrades host
1347	 * userspace indirect branch performance.
1348	 */
1349	if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) ||
1350	    (cpu_has(c, X86_FEATURE_AUTOIBRS) &&
1351	     !cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
1352		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1353		if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1354		    !(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
1355			setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1356	}
1357
1358	if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1359	    !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
1360		setup_force_cpu_bug(X86_BUG_MDS);
1361		if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1362			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1363	}
1364
1365	if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1366		setup_force_cpu_bug(X86_BUG_SWAPGS);
1367
1368	/*
1369	 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1370	 *	- TSX is supported or
1371	 *	- TSX_CTRL is present
1372	 *
1373	 * TSX_CTRL check is needed for cases when TSX could be disabled before
1374	 * the kernel boot e.g. kexec.
1375	 * TSX_CTRL check alone is not sufficient for cases when the microcode
1376	 * update is not present or running as guest that don't get TSX_CTRL.
1377	 */
1378	if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
1379	    (cpu_has(c, X86_FEATURE_RTM) ||
1380	     (x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
1381		setup_force_cpu_bug(X86_BUG_TAA);
1382
1383	/*
1384	 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1385	 * in the vulnerability blacklist.
1386	 *
1387	 * Some of the implications and mitigation of Shared Buffers Data
1388	 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1389	 * SRBDS.
1390	 */
1391	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1392	     cpu_has(c, X86_FEATURE_RDSEED)) &&
1393	    cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1394		    setup_force_cpu_bug(X86_BUG_SRBDS);
1395
1396	/*
1397	 * Processor MMIO Stale Data bug enumeration
1398	 *
1399	 * Affected CPU list is generally enough to enumerate the vulnerability,
1400	 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1401	 * not want the guest to enumerate the bug.
1402	 *
1403	 * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1404	 * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1405	 */
1406	if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
1407		if (cpu_matches(cpu_vuln_blacklist, MMIO))
1408			setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1409		else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1410			setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1411	}
1412
1413	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1414		if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
1415			setup_force_cpu_bug(X86_BUG_RETBLEED);
1416	}
1417
 
 
 
 
 
1418	if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
1419		setup_force_cpu_bug(X86_BUG_SMT_RSB);
1420
1421	if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1422		if (cpu_matches(cpu_vuln_blacklist, SRSO))
1423			setup_force_cpu_bug(X86_BUG_SRSO);
1424	}
1425
1426	/*
1427	 * Check if CPU is vulnerable to GDS. If running in a virtual machine on
1428	 * an affected processor, the VMM may have disabled the use of GATHER by
1429	 * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1430	 * which means that AVX will be disabled.
1431	 */
1432	if (cpu_matches(cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
1433	    boot_cpu_has(X86_FEATURE_AVX))
1434		setup_force_cpu_bug(X86_BUG_GDS);
1435
1436	if (vulnerable_to_rfds(x86_arch_cap_msr))
1437		setup_force_cpu_bug(X86_BUG_RFDS);
1438
1439	/* When virtualized, eIBRS could be hidden, assume vulnerable */
1440	if (!(x86_arch_cap_msr & ARCH_CAP_BHI_NO) &&
1441	    !cpu_matches(cpu_vuln_whitelist, NO_BHI) &&
1442	    (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
1443	     boot_cpu_has(X86_FEATURE_HYPERVISOR)))
1444		setup_force_cpu_bug(X86_BUG_BHI);
1445
1446	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1447		return;
1448
1449	/* Rogue Data Cache Load? No! */
1450	if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
1451		return;
1452
1453	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1454
1455	if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1456		return;
1457
1458	setup_force_cpu_bug(X86_BUG_L1TF);
1459}
1460
1461/*
1462 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1463 * unfortunately, that's not true in practice because of early VIA
1464 * chips and (more importantly) broken virtualizers that are not easy
1465 * to detect. In the latter case it doesn't even *fail* reliably, so
1466 * probing for it doesn't even work. Disable it completely on 32-bit
1467 * unless we can find a reliable way to detect all the broken cases.
1468 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1469 */
1470static void detect_nopl(void)
1471{
1472#ifdef CONFIG_X86_32
1473	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1474#else
1475	setup_force_cpu_cap(X86_FEATURE_NOPL);
1476#endif
1477}
1478
1479/*
1480 * We parse cpu parameters early because fpu__init_system() is executed
1481 * before parse_early_param().
1482 */
1483static void __init cpu_parse_early_param(void)
1484{
1485	char arg[128];
1486	char *argptr = arg, *opt;
1487	int arglen, taint = 0;
1488
1489#ifdef CONFIG_X86_32
1490	if (cmdline_find_option_bool(boot_command_line, "no387"))
1491#ifdef CONFIG_MATH_EMULATION
1492		setup_clear_cpu_cap(X86_FEATURE_FPU);
1493#else
1494		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1495#endif
1496
1497	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1498		setup_clear_cpu_cap(X86_FEATURE_FXSR);
1499#endif
1500
1501	if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1502		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1503
1504	if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1505		setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1506
1507	if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1508		setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1509
1510	if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1511		setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1512
1513	arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1514	if (arglen <= 0)
1515		return;
1516
1517	pr_info("Clearing CPUID bits:");
1518
1519	while (argptr) {
1520		bool found __maybe_unused = false;
1521		unsigned int bit;
1522
1523		opt = strsep(&argptr, ",");
1524
1525		/*
1526		 * Handle naked numbers first for feature flags which don't
1527		 * have names.
1528		 */
1529		if (!kstrtouint(opt, 10, &bit)) {
1530			if (bit < NCAPINTS * 32) {
1531
 
1532				/* empty-string, i.e., ""-defined feature flags */
1533				if (!x86_cap_flags[bit])
1534					pr_cont(" " X86_CAP_FMT_NUM, x86_cap_flag_num(bit));
1535				else
 
1536					pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1537
1538				setup_clear_cpu_cap(bit);
1539				taint++;
1540			}
1541			/*
1542			 * The assumption is that there are no feature names with only
1543			 * numbers in the name thus go to the next argument.
1544			 */
1545			continue;
1546		}
1547
 
1548		for (bit = 0; bit < 32 * NCAPINTS; bit++) {
1549			if (!x86_cap_flag(bit))
1550				continue;
1551
1552			if (strcmp(x86_cap_flag(bit), opt))
1553				continue;
1554
1555			pr_cont(" %s", opt);
1556			setup_clear_cpu_cap(bit);
1557			taint++;
1558			found = true;
1559			break;
1560		}
1561
1562		if (!found)
1563			pr_cont(" (unknown: %s)", opt);
 
1564	}
1565	pr_cont("\n");
1566
1567	if (taint)
1568		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1569}
1570
1571/*
1572 * Do minimum CPU detection early.
1573 * Fields really needed: vendor, cpuid_level, family, model, mask,
1574 * cache alignment.
1575 * The others are not touched to avoid unwanted side effects.
1576 *
1577 * WARNING: this function is only called on the boot CPU.  Don't add code
1578 * here that is supposed to run on all CPUs.
1579 */
1580static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1581{
 
 
 
 
 
 
 
 
 
 
 
1582	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1583	c->extended_cpuid_level = 0;
1584
1585	if (!have_cpuid_p())
1586		identify_cpu_without_cpuid(c);
1587
1588	/* cyrix could have cpuid enabled via c_identify()*/
1589	if (have_cpuid_p()) {
1590		cpu_detect(c);
1591		get_cpu_vendor(c);
1592		intel_unlock_cpuid_leafs(c);
1593		get_cpu_cap(c);
1594		setup_force_cpu_cap(X86_FEATURE_CPUID);
1595		get_cpu_address_sizes(c);
 
1596		cpu_parse_early_param();
1597
1598		cpu_init_topology(c);
1599
1600		if (this_cpu->c_early_init)
1601			this_cpu->c_early_init(c);
1602
1603		c->cpu_index = 0;
1604		filter_cpuid_features(c, false);
1605
1606		if (this_cpu->c_bsp_init)
1607			this_cpu->c_bsp_init(c);
1608	} else {
1609		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1610		get_cpu_address_sizes(c);
1611		cpu_init_topology(c);
1612	}
1613
1614	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1615
1616	cpu_set_bug_bits(c);
1617
1618	sld_setup(c);
1619
 
 
 
 
1620#ifdef CONFIG_X86_32
1621	/*
1622	 * Regardless of whether PCID is enumerated, the SDM says
1623	 * that it can't be enabled in 32-bit mode.
1624	 */
1625	setup_clear_cpu_cap(X86_FEATURE_PCID);
1626#endif
1627
1628	/*
1629	 * Later in the boot process pgtable_l5_enabled() relies on
1630	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1631	 * enabled by this point we need to clear the feature bit to avoid
1632	 * false-positives at the later stage.
1633	 *
1634	 * pgtable_l5_enabled() can be false here for several reasons:
1635	 *  - 5-level paging is disabled compile-time;
1636	 *  - it's 32-bit kernel;
1637	 *  - machine doesn't support 5-level paging;
1638	 *  - user specified 'no5lvl' in kernel command line.
1639	 */
1640	if (!pgtable_l5_enabled())
1641		setup_clear_cpu_cap(X86_FEATURE_LA57);
1642
1643	detect_nopl();
1644}
1645
1646void __init early_cpu_init(void)
1647{
1648	const struct cpu_dev *const *cdev;
1649	int count = 0;
1650
1651#ifdef CONFIG_PROCESSOR_SELECT
1652	pr_info("KERNEL supported cpus:\n");
1653#endif
1654
1655	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1656		const struct cpu_dev *cpudev = *cdev;
1657
1658		if (count >= X86_VENDOR_NUM)
1659			break;
1660		cpu_devs[count] = cpudev;
1661		count++;
1662
1663#ifdef CONFIG_PROCESSOR_SELECT
1664		{
1665			unsigned int j;
1666
1667			for (j = 0; j < 2; j++) {
1668				if (!cpudev->c_ident[j])
1669					continue;
1670				pr_info("  %s %s\n", cpudev->c_vendor,
1671					cpudev->c_ident[j]);
1672			}
1673		}
1674#endif
1675	}
1676	early_identify_cpu(&boot_cpu_data);
1677}
1678
1679static bool detect_null_seg_behavior(void)
1680{
1681	/*
1682	 * Empirically, writing zero to a segment selector on AMD does
1683	 * not clear the base, whereas writing zero to a segment
1684	 * selector on Intel does clear the base.  Intel's behavior
1685	 * allows slightly faster context switches in the common case
1686	 * where GS is unused by the prev and next threads.
1687	 *
1688	 * Since neither vendor documents this anywhere that I can see,
1689	 * detect it directly instead of hard-coding the choice by
1690	 * vendor.
1691	 *
1692	 * I've designated AMD's behavior as the "bug" because it's
1693	 * counterintuitive and less friendly.
1694	 */
1695
1696	unsigned long old_base, tmp;
1697	rdmsrl(MSR_FS_BASE, old_base);
1698	wrmsrl(MSR_FS_BASE, 1);
1699	loadsegment(fs, 0);
1700	rdmsrl(MSR_FS_BASE, tmp);
1701	wrmsrl(MSR_FS_BASE, old_base);
1702	return tmp == 0;
1703}
1704
1705void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1706{
1707	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1708	if (!IS_ENABLED(CONFIG_X86_64))
1709		return;
1710
1711	if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
 
 
1712		return;
1713
1714	/*
1715	 * CPUID bit above wasn't set. If this kernel is still running
1716	 * as a HV guest, then the HV has decided not to advertize
1717	 * that CPUID bit for whatever reason.	For example, one
1718	 * member of the migration pool might be vulnerable.  Which
1719	 * means, the bug is present: set the BUG flag and return.
1720	 */
1721	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1722		set_cpu_bug(c, X86_BUG_NULL_SEG);
1723		return;
1724	}
1725
1726	/*
1727	 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1728	 * 0x18 is the respective family for Hygon.
1729	 */
1730	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1731	    detect_null_seg_behavior())
1732		return;
1733
1734	/* All the remaining ones are affected */
1735	set_cpu_bug(c, X86_BUG_NULL_SEG);
1736}
1737
1738static void generic_identify(struct cpuinfo_x86 *c)
1739{
1740	c->extended_cpuid_level = 0;
1741
1742	if (!have_cpuid_p())
1743		identify_cpu_without_cpuid(c);
1744
1745	/* cyrix could have cpuid enabled via c_identify()*/
1746	if (!have_cpuid_p())
1747		return;
1748
1749	cpu_detect(c);
1750
1751	get_cpu_vendor(c);
1752	intel_unlock_cpuid_leafs(c);
1753	get_cpu_cap(c);
1754
1755	get_cpu_address_sizes(c);
1756
 
 
 
 
 
 
 
 
 
 
 
 
1757	get_model_name(c); /* Default name */
1758
1759	/*
1760	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1761	 * systems that run Linux at CPL > 0 may or may not have the
1762	 * issue, but, even if they have the issue, there's absolutely
1763	 * nothing we can do about it because we can't use the real IRET
1764	 * instruction.
1765	 *
1766	 * NB: For the time being, only 32-bit kernels support
1767	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1768	 * whether to apply espfix using paravirt hooks.  If any
1769	 * non-paravirt system ever shows up that does *not* have the
1770	 * ESPFIX issue, we can change this.
1771	 */
1772#ifdef CONFIG_X86_32
1773	set_cpu_bug(c, X86_BUG_ESPFIX);
1774#endif
1775}
1776
1777/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1778 * This does the hard work of actually picking apart the CPU stuff...
1779 */
1780static void identify_cpu(struct cpuinfo_x86 *c)
1781{
1782	int i;
1783
1784	c->loops_per_jiffy = loops_per_jiffy;
1785	c->x86_cache_size = 0;
1786	c->x86_vendor = X86_VENDOR_UNKNOWN;
1787	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
1788	c->x86_vendor_id[0] = '\0'; /* Unset */
1789	c->x86_model_id[0] = '\0';  /* Unset */
 
 
 
1790#ifdef CONFIG_X86_64
1791	c->x86_clflush_size = 64;
1792	c->x86_phys_bits = 36;
1793	c->x86_virt_bits = 48;
1794#else
1795	c->cpuid_level = -1;	/* CPUID not detected */
1796	c->x86_clflush_size = 32;
1797	c->x86_phys_bits = 32;
1798	c->x86_virt_bits = 32;
1799#endif
1800	c->x86_cache_alignment = c->x86_clflush_size;
1801	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1802#ifdef CONFIG_X86_VMX_FEATURE_NAMES
1803	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1804#endif
1805
1806	generic_identify(c);
1807
1808	cpu_parse_topology(c);
1809
1810	if (this_cpu->c_identify)
1811		this_cpu->c_identify(c);
1812
1813	/* Clear/Set all flags overridden by options, after probe */
1814	apply_forced_caps(c);
1815
1816	/*
1817	 * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1818	 * Hygon will clear it in ->c_init() below.
1819	 */
1820	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1821
1822	/*
1823	 * Vendor-specific initialization.  In this section we
1824	 * canonicalize the feature flags, meaning if there are
1825	 * features a certain CPU supports which CPUID doesn't
1826	 * tell us, CPUID claiming incorrect flags, or other bugs,
1827	 * we handle them here.
1828	 *
1829	 * At the end of this section, c->x86_capability better
1830	 * indicate the features this CPU genuinely supports!
1831	 */
1832	if (this_cpu->c_init)
1833		this_cpu->c_init(c);
1834
1835	/* Disable the PN if appropriate */
1836	squash_the_stupid_serial_number(c);
1837
1838	/* Set up SMEP/SMAP/UMIP */
1839	setup_smep(c);
1840	setup_smap(c);
1841	setup_umip(c);
1842
1843	/* Enable FSGSBASE instructions if available. */
1844	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1845		cr4_set_bits(X86_CR4_FSGSBASE);
1846		elf_hwcap2 |= HWCAP2_FSGSBASE;
1847	}
1848
1849	/*
1850	 * The vendor-specific functions might have changed features.
1851	 * Now we do "generic changes."
1852	 */
1853
1854	/* Filter out anything that depends on CPUID levels we don't have */
1855	filter_cpuid_features(c, true);
1856
1857	/* If the model name is still unset, do table lookup. */
1858	if (!c->x86_model_id[0]) {
1859		const char *p;
1860		p = table_lookup_model(c);
1861		if (p)
1862			strcpy(c->x86_model_id, p);
1863		else
1864			/* Last resort... */
1865			sprintf(c->x86_model_id, "%02x/%02x",
1866				c->x86, c->x86_model);
1867	}
1868
 
 
 
 
1869	x86_init_rdrand(c);
1870	setup_pku(c);
1871	setup_cet(c);
1872
1873	/*
1874	 * Clear/Set all flags overridden by options, need do it
1875	 * before following smp all cpus cap AND.
1876	 */
1877	apply_forced_caps(c);
1878
1879	/*
1880	 * On SMP, boot_cpu_data holds the common feature set between
1881	 * all CPUs; so make sure that we indicate which features are
1882	 * common between the CPUs.  The first time this routine gets
1883	 * executed, c == &boot_cpu_data.
1884	 */
1885	if (c != &boot_cpu_data) {
1886		/* AND the already accumulated flags with these */
1887		for (i = 0; i < NCAPINTS; i++)
1888			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1889
1890		/* OR, i.e. replicate the bug flags */
1891		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1892			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1893	}
1894
1895	ppin_init(c);
1896
1897	/* Init Machine Check Exception if available. */
1898	mcheck_cpu_init(c);
1899
 
 
1900#ifdef CONFIG_NUMA
1901	numa_add_cpu(smp_processor_id());
1902#endif
1903}
1904
1905/*
1906 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1907 * on 32-bit kernels:
1908 */
1909#ifdef CONFIG_X86_32
1910void enable_sep_cpu(void)
1911{
1912	struct tss_struct *tss;
1913	int cpu;
1914
1915	if (!boot_cpu_has(X86_FEATURE_SEP))
1916		return;
1917
1918	cpu = get_cpu();
1919	tss = &per_cpu(cpu_tss_rw, cpu);
1920
1921	/*
1922	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1923	 * see the big comment in struct x86_hw_tss's definition.
1924	 */
1925
1926	tss->x86_tss.ss1 = __KERNEL_CS;
1927	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1928	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1929	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1930
1931	put_cpu();
1932}
1933#endif
1934
1935static __init void identify_boot_cpu(void)
1936{
1937	identify_cpu(&boot_cpu_data);
1938	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
1939		pr_info("CET detected: Indirect Branch Tracking enabled\n");
1940#ifdef CONFIG_X86_32
 
1941	enable_sep_cpu();
1942#endif
1943	cpu_detect_tlb(&boot_cpu_data);
1944	setup_cr_pinning();
1945
1946	tsx_init();
1947	tdx_init();
1948	lkgs_init();
1949}
1950
1951void identify_secondary_cpu(struct cpuinfo_x86 *c)
1952{
1953	BUG_ON(c == &boot_cpu_data);
1954	identify_cpu(c);
1955#ifdef CONFIG_X86_32
1956	enable_sep_cpu();
1957#endif
 
1958	x86_spec_ctrl_setup_ap();
1959	update_srbds_msr();
1960	if (boot_cpu_has_bug(X86_BUG_GDS))
1961		update_gds_msr();
1962
1963	tsx_ap_init();
1964}
1965
1966void print_cpu_info(struct cpuinfo_x86 *c)
1967{
1968	const char *vendor = NULL;
1969
1970	if (c->x86_vendor < X86_VENDOR_NUM) {
1971		vendor = this_cpu->c_vendor;
1972	} else {
1973		if (c->cpuid_level >= 0)
1974			vendor = c->x86_vendor_id;
1975	}
1976
1977	if (vendor && !strstr(c->x86_model_id, vendor))
1978		pr_cont("%s ", vendor);
1979
1980	if (c->x86_model_id[0])
1981		pr_cont("%s", c->x86_model_id);
1982	else
1983		pr_cont("%d86", c->x86);
1984
1985	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1986
1987	if (c->x86_stepping || c->cpuid_level >= 0)
1988		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
1989	else
1990		pr_cont(")\n");
1991}
1992
1993/*
1994 * clearcpuid= was already parsed in cpu_parse_early_param().  This dummy
1995 * function prevents it from becoming an environment variable for init.
1996 */
1997static __init int setup_clearcpuid(char *arg)
1998{
1999	return 1;
2000}
2001__setup("clearcpuid=", setup_clearcpuid);
2002
2003DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
2004	.current_task	= &init_task,
2005	.preempt_count	= INIT_PREEMPT_COUNT,
2006	.top_of_stack	= TOP_OF_INIT_STACK,
2007};
2008EXPORT_PER_CPU_SYMBOL(pcpu_hot);
2009EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
2010
2011#ifdef CONFIG_X86_64
2012DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
2013		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
2014EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2015
2016static void wrmsrl_cstar(unsigned long val)
2017{
2018	/*
2019	 * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2020	 * is so far ignored by the CPU, but raises a #VE trap in a TDX
2021	 * guest. Avoid the pointless write on all Intel CPUs.
2022	 */
2023	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2024		wrmsrl(MSR_CSTAR, val);
2025}
2026
2027static inline void idt_syscall_init(void)
 
2028{
 
2029	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2030
2031	if (ia32_enabled()) {
2032		wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2033		/*
2034		 * This only works on Intel CPUs.
2035		 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2036		 * This does not cause SYSENTER to jump to the wrong location, because
2037		 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2038		 */
2039		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2040		wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2041			    (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2042		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2043	} else {
2044		wrmsrl_cstar((unsigned long)entry_SYSCALL32_ignore);
2045		wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2046		wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2047		wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2048	}
2049
2050	/*
2051	 * Flags to clear on syscall; clear as much as possible
2052	 * to minimize user space-kernel interference.
2053	 */
2054	wrmsrl(MSR_SYSCALL_MASK,
2055	       X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2056	       X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2057	       X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2058	       X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2059	       X86_EFLAGS_AC|X86_EFLAGS_ID);
2060}
2061
2062/* May not be marked __init: used by software suspend */
2063void syscall_init(void)
2064{
2065	/* The default user and kernel segments */
2066	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2067
2068	/*
2069	 * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
2070	 * SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
2071	 * entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
2072	 * instruction to return to ring 3 (both sysexit and sysret cause
2073	 * #UD when FRED is enabled).
2074	 */
2075	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2076		idt_syscall_init();
2077}
2078
2079#else	/* CONFIG_X86_64 */
2080
2081#ifdef CONFIG_STACKPROTECTOR
2082DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2083EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2084#endif
2085
2086#endif	/* CONFIG_X86_64 */
2087
2088/*
2089 * Clear all 6 debug registers:
2090 */
2091static void clear_all_debug_regs(void)
2092{
2093	int i;
2094
2095	for (i = 0; i < 8; i++) {
2096		/* Ignore db4, db5 */
2097		if ((i == 4) || (i == 5))
2098			continue;
2099
2100		set_debugreg(0, i);
2101	}
2102}
2103
2104#ifdef CONFIG_KGDB
2105/*
2106 * Restore debug regs if using kgdbwait and you have a kernel debugger
2107 * connection established.
2108 */
2109static void dbg_restore_debug_regs(void)
2110{
2111	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2112		arch_kgdb_ops.correct_hw_break();
2113}
2114#else /* ! CONFIG_KGDB */
2115#define dbg_restore_debug_regs()
2116#endif /* ! CONFIG_KGDB */
2117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2118static inline void setup_getcpu(int cpu)
2119{
2120	unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2121	struct desc_struct d = { };
2122
2123	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2124		wrmsr(MSR_TSC_AUX, cpudata, 0);
2125
2126	/* Store CPU and node number in limit. */
2127	d.limit0 = cpudata;
2128	d.limit1 = cpudata >> 16;
2129
2130	d.type = 5;		/* RO data, expand down, accessed */
2131	d.dpl = 3;		/* Visible to user code */
2132	d.s = 1;		/* Not a system segment */
2133	d.p = 1;		/* Present */
2134	d.d = 1;		/* 32-bit */
2135
2136	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2137}
2138
2139#ifdef CONFIG_X86_64
 
 
 
 
 
2140static inline void tss_setup_ist(struct tss_struct *tss)
2141{
2142	/* Set up the per-CPU TSS IST stacks */
2143	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2144	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2145	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2146	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2147	/* Only mapped when SEV-ES is active */
2148	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2149}
 
2150#else /* CONFIG_X86_64 */
 
 
 
 
 
 
 
 
2151static inline void tss_setup_ist(struct tss_struct *tss) { }
 
2152#endif /* !CONFIG_X86_64 */
2153
2154static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2155{
2156	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2157
2158#ifdef CONFIG_X86_IOPL_IOPERM
2159	tss->io_bitmap.prev_max = 0;
2160	tss->io_bitmap.prev_sequence = 0;
2161	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2162	/*
2163	 * Invalidate the extra array entry past the end of the all
2164	 * permission bitmap as required by the hardware.
2165	 */
2166	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2167#endif
2168}
2169
2170/*
2171 * Setup everything needed to handle exceptions from the IDT, including the IST
2172 * exceptions which use paranoid_entry().
2173 */
2174void cpu_init_exception_handling(void)
2175{
2176	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2177	int cpu = raw_smp_processor_id();
2178
2179	/* paranoid_entry() gets the CPU number from the GDT */
2180	setup_getcpu(cpu);
2181
2182	/* For IDT mode, IST vectors need to be set in TSS. */
2183	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2184		tss_setup_ist(tss);
2185	tss_setup_io_bitmap(tss);
2186	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2187
2188	load_TR_desc();
2189
2190	/* GHCB needs to be setup to handle #VC. */
2191	setup_ghcb();
2192
2193	if (cpu_feature_enabled(X86_FEATURE_FRED))
2194		cpu_init_fred_exceptions();
2195	else
2196		load_current_idt();
2197}
2198
2199/*
2200 * cpu_init() initializes state that is per-CPU. Some data is already
2201 * initialized (naturally) in the bootstrap process, such as the GDT.  We
2202 * reload it nevertheless, this function acts as a 'CPU state barrier',
2203 * nothing should get across.
2204 */
2205void cpu_init(void)
2206{
2207	struct task_struct *cur = current;
2208	int cpu = raw_smp_processor_id();
2209
 
 
 
 
2210#ifdef CONFIG_NUMA
2211	if (this_cpu_read(numa_node) == 0 &&
2212	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
2213		set_numa_node(early_cpu_to_node(cpu));
2214#endif
2215	pr_debug("Initializing CPU#%d\n", cpu);
2216
2217	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2218	    boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2219		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2220
2221	if (IS_ENABLED(CONFIG_X86_64)) {
2222		loadsegment(fs, 0);
2223		memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2224		syscall_init();
2225
2226		wrmsrl(MSR_FS_BASE, 0);
2227		wrmsrl(MSR_KERNEL_GS_BASE, 0);
2228		barrier();
2229
2230		x2apic_setup();
2231	}
2232
2233	mmgrab(&init_mm);
2234	cur->active_mm = &init_mm;
2235	BUG_ON(cur->mm);
2236	initialize_tlbstate_and_flush();
2237	enter_lazy_tlb(&init_mm, cur);
2238
2239	/*
2240	 * sp0 points to the entry trampoline stack regardless of what task
2241	 * is running.
2242	 */
2243	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2244
2245	load_mm_ldt(&init_mm);
2246
2247	clear_all_debug_regs();
2248	dbg_restore_debug_regs();
2249
2250	doublefault_init_cpu_tss();
2251
 
 
2252	if (is_uv_system())
2253		uv_cpu_init();
2254
2255	load_fixmap_gdt(cpu);
2256}
2257
2258#ifdef CONFIG_MICROCODE_LATE_LOADING
2259/**
2260 * store_cpu_caps() - Store a snapshot of CPU capabilities
2261 * @curr_info: Pointer where to store it
2262 *
2263 * Returns: None
2264 */
2265void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2266{
2267	/* Reload CPUID max function as it might've changed. */
2268	curr_info->cpuid_level = cpuid_eax(0);
2269
2270	/* Copy all capability leafs and pick up the synthetic ones. */
2271	memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2272	       sizeof(curr_info->x86_capability));
2273
2274	/* Get the hardware CPUID leafs */
2275	get_cpu_cap(curr_info);
2276}
 
2277
2278/**
2279 * microcode_check() - Check if any CPU capabilities changed after an update.
2280 * @prev_info:	CPU capabilities stored before an update.
2281 *
2282 * The microcode loader calls this upon late microcode load to recheck features,
2283 * only when microcode has been updated. Caller holds and CPU hotplug lock.
2284 *
2285 * Return: None
2286 */
2287void microcode_check(struct cpuinfo_x86 *prev_info)
2288{
2289	struct cpuinfo_x86 curr_info;
2290
2291	perf_check_microcode();
2292
2293	amd_check_microcode();
 
2294
2295	store_cpu_caps(&curr_info);
 
 
 
 
 
2296
2297	if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
2298		    sizeof(prev_info->x86_capability)))
 
2299		return;
2300
2301	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2302	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2303}
2304#endif
2305
2306/*
2307 * Invoked from core CPU hotplug code after hotplug operations
2308 */
2309void arch_smt_update(void)
2310{
2311	/* Handle the speculative execution misfeatures */
2312	cpu_bugs_smt_update();
2313	/* Check whether IPI broadcasting can be enabled */
2314	apic_smt_update();
2315}
2316
2317void __init arch_cpu_finalize_init(void)
2318{
2319	struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
2320
2321	identify_boot_cpu();
2322
2323	select_idle_routine();
2324
2325	/*
2326	 * identify_boot_cpu() initialized SMT support information, let the
2327	 * core code know.
2328	 */
2329	cpu_smt_set_num_threads(__max_threads_per_core, __max_threads_per_core);
2330
2331	if (!IS_ENABLED(CONFIG_SMP)) {
2332		pr_info("CPU: ");
2333		print_cpu_info(&boot_cpu_data);
2334	}
2335
2336	cpu_select_mitigations();
2337
2338	arch_smt_update();
2339
2340	if (IS_ENABLED(CONFIG_X86_32)) {
2341		/*
2342		 * Check whether this is a real i386 which is not longer
2343		 * supported and fixup the utsname.
2344		 */
2345		if (boot_cpu_data.x86 < 4)
2346			panic("Kernel requires i486+ for 'invlpg' and other features");
2347
2348		init_utsname()->machine[1] =
2349			'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2350	}
2351
2352	/*
2353	 * Must be before alternatives because it might set or clear
2354	 * feature bits.
2355	 */
2356	fpu__init_system();
2357	fpu__init_cpu();
2358
2359	/*
2360	 * Ensure that access to the per CPU representation has the initial
2361	 * boot CPU configuration.
2362	 */
2363	*c = boot_cpu_data;
2364	c->initialized = true;
2365
2366	alternative_instructions();
2367
2368	if (IS_ENABLED(CONFIG_X86_64)) {
2369		/*
2370		 * Make sure the first 2MB area is not mapped by huge pages
2371		 * There are typically fixed size MTRRs in there and overlapping
2372		 * MTRRs into large pages causes slow downs.
2373		 *
2374		 * Right now we don't do that with gbpages because there seems
2375		 * very little benefit for that case.
2376		 */
2377		if (!direct_gbpages)
2378			set_memory_4k((unsigned long)__va(0), 1);
2379	} else {
2380		fpu__init_check_bugs();
2381	}
2382
2383	/*
2384	 * This needs to be called before any devices perform DMA
2385	 * operations that might use the SWIOTLB bounce buffers. It will
2386	 * mark the bounce buffers as decrypted so that their usage will
2387	 * not cause "plain-text" data to be decrypted when accessed. It
2388	 * must be called after late_time_init() so that Hyper-V x86/x64
2389	 * hypercalls work when the SWIOTLB bounce buffers are decrypted.
2390	 */
2391	mem_encrypt_init();
2392}