1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Kernel-based Virtual Machine driver for Linux
   4 * cpuid support routines
   5 *
   6 * derived from arch/x86/kvm/x86.c
   7 *
   8 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
   9 * Copyright IBM Corporation, 2008
  10 */
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13#include <linux/kvm_host.h>
  14#include "linux/lockdep.h"
  15#include <linux/export.h>
  16#include <linux/vmalloc.h>
  17#include <linux/uaccess.h>
  18#include <linux/sched/stat.h>
  19
  20#include <asm/processor.h>
  21#include <asm/user.h>
  22#include <asm/fpu/xstate.h>
  23#include <asm/sgx.h>
  24#include <asm/cpuid.h>
  25#include "cpuid.h"
  26#include "lapic.h"
  27#include "mmu.h"
  28#include "trace.h"
  29#include "pmu.h"
  30#include "xen.h"
  31
  32/*
  33 * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
  34 * aligned to sizeof(unsigned long) because it's not accessed via bitops.
  35 */
  36u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly;
  37EXPORT_SYMBOL_GPL(kvm_cpu_caps);
  38
  39u32 xstate_required_size(u64 xstate_bv, bool compacted)
  40{
  41	int feature_bit = 0;
  42	u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
  43
  44	xstate_bv &= XFEATURE_MASK_EXTEND;
  45	while (xstate_bv) {
  46		if (xstate_bv & 0x1) {
  47		        u32 eax, ebx, ecx, edx, offset;
  48		        cpuid_count(0xD, feature_bit, &eax, &ebx, &ecx, &edx);
  49			/* ECX[1]: 64B alignment in compacted form */
  50			if (compacted)
  51				offset = (ecx & 0x2) ? ALIGN(ret, 64) : ret;
  52			else
  53				offset = ebx;
  54			ret = max(ret, offset + eax);
  55		}
  56
  57		xstate_bv >>= 1;
  58		feature_bit++;
  59	}
  60
  61	return ret;
  62}
  63
  64#define F feature_bit
  65
  66/* Scattered Flag - For features that are scattered by cpufeatures.h. */
  67#define SF(name)						\
  68({								\
  69	BUILD_BUG_ON(X86_FEATURE_##name >= MAX_CPU_FEATURES);	\
  70	(boot_cpu_has(X86_FEATURE_##name) ? F(name) : 0);	\
  71})
  72
  73/*
  74 * Magic value used by KVM when querying userspace-provided CPUID entries and
  75 * doesn't care about the CPIUD index because the index of the function in
  76 * question is not significant.  Note, this magic value must have at least one
  77 * bit set in bits[63:32] and must be consumed as a u64 by cpuid_entry2_find()
  78 * to avoid false positives when processing guest CPUID input.
  79 */
  80#define KVM_CPUID_INDEX_NOT_SIGNIFICANT -1ull
  81
  82static inline struct kvm_cpuid_entry2 *cpuid_entry2_find(
  83	struct kvm_cpuid_entry2 *entries, int nent, u32 function, u64 index)
  84{
  85	struct kvm_cpuid_entry2 *e;
  86	int i;
  87
  88	/*
  89	 * KVM has a semi-arbitrary rule that querying the guest's CPUID model
  90	 * with IRQs disabled is disallowed.  The CPUID model can legitimately
  91	 * have over one hundred entries, i.e. the lookup is slow, and IRQs are
  92	 * typically disabled in KVM only when KVM is in a performance critical
  93	 * path, e.g. the core VM-Enter/VM-Exit run loop.  Nothing will break
  94	 * if this rule is violated, this assertion is purely to flag potential
  95	 * performance issues.  If this fires, consider moving the lookup out
  96	 * of the hotpath, e.g. by caching information during CPUID updates.
  97	 */
  98	lockdep_assert_irqs_enabled();
  99
 100	for (i = 0; i < nent; i++) {
 101		e = &entries[i];
 102
 103		if (e->function != function)
 104			continue;
 105
 106		/*
 107		 * If the index isn't significant, use the first entry with a
 108		 * matching function.  It's userspace's responsibility to not
 109		 * provide "duplicate" entries in all cases.
 110		 */
 111		if (!(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) || e->index == index)
 112			return e;
 113
 114
 115		/*
 116		 * Similarly, use the first matching entry if KVM is doing a
 117		 * lookup (as opposed to emulating CPUID) for a function that's
 118		 * architecturally defined as not having a significant index.
 119		 */
 120		if (index == KVM_CPUID_INDEX_NOT_SIGNIFICANT) {
 121			/*
 122			 * Direct lookups from KVM should not diverge from what
 123			 * KVM defines internally (the architectural behavior).
 124			 */
 125			WARN_ON_ONCE(cpuid_function_is_indexed(function));
 126			return e;
 127		}
 128	}
 129
 130	return NULL;
 131}
 132
 133static int kvm_check_cpuid(struct kvm_vcpu *vcpu,
 134			   struct kvm_cpuid_entry2 *entries,
 135			   int nent)
 136{
 137	struct kvm_cpuid_entry2 *best;
 138	u64 xfeatures;
 139
 140	/*
 141	 * The existing code assumes virtual address is 48-bit or 57-bit in the
 142	 * canonical address checks; exit if it is ever changed.
 143	 */
 144	best = cpuid_entry2_find(entries, nent, 0x80000008,
 145				 KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 146	if (best) {
 147		int vaddr_bits = (best->eax & 0xff00) >> 8;
 148
 149		if (vaddr_bits != 48 && vaddr_bits != 57 && vaddr_bits != 0)
 150			return -EINVAL;
 151	}
 152
 153	/*
 154	 * Exposing dynamic xfeatures to the guest requires additional
 155	 * enabling in the FPU, e.g. to expand the guest XSAVE state size.
 156	 */
 157	best = cpuid_entry2_find(entries, nent, 0xd, 0);
 158	if (!best)
 159		return 0;
 160
 161	xfeatures = best->eax | ((u64)best->edx << 32);
 162	xfeatures &= XFEATURE_MASK_USER_DYNAMIC;
 163	if (!xfeatures)
 164		return 0;
 165
 166	return fpu_enable_guest_xfd_features(&vcpu->arch.guest_fpu, xfeatures);
 167}
 168
 169/* Check whether the supplied CPUID data is equal to what is already set for the vCPU. */
 170static int kvm_cpuid_check_equal(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
 171				 int nent)
 172{
 173	struct kvm_cpuid_entry2 *orig;
 174	int i;
 175
 176	if (nent != vcpu->arch.cpuid_nent)
 177		return -EINVAL;
 178
 179	for (i = 0; i < nent; i++) {
 180		orig = &vcpu->arch.cpuid_entries[i];
 181		if (e2[i].function != orig->function ||
 182		    e2[i].index != orig->index ||
 183		    e2[i].flags != orig->flags ||
 184		    e2[i].eax != orig->eax || e2[i].ebx != orig->ebx ||
 185		    e2[i].ecx != orig->ecx || e2[i].edx != orig->edx)
 186			return -EINVAL;
 187	}
 188
 189	return 0;
 190}
 191
 192static struct kvm_hypervisor_cpuid kvm_get_hypervisor_cpuid(struct kvm_vcpu *vcpu,
 193							    const char *sig)
 194{
 195	struct kvm_hypervisor_cpuid cpuid = {};
 196	struct kvm_cpuid_entry2 *entry;
 197	u32 base;
 198
 199	for_each_possible_hypervisor_cpuid_base(base) {
 200		entry = kvm_find_cpuid_entry(vcpu, base);
 201
 202		if (entry) {
 203			u32 signature[3];
 204
 205			signature[0] = entry->ebx;
 206			signature[1] = entry->ecx;
 207			signature[2] = entry->edx;
 208
 209			if (!memcmp(signature, sig, sizeof(signature))) {
 210				cpuid.base = base;
 211				cpuid.limit = entry->eax;
 212				break;
 213			}
 214		}
 215	}
 216
 217	return cpuid;
 218}
 219
 220static struct kvm_cpuid_entry2 *__kvm_find_kvm_cpuid_features(struct kvm_vcpu *vcpu,
 221					      struct kvm_cpuid_entry2 *entries, int nent)
 222{
 223	u32 base = vcpu->arch.kvm_cpuid.base;
 224
 225	if (!base)
 226		return NULL;
 227
 228	return cpuid_entry2_find(entries, nent, base | KVM_CPUID_FEATURES,
 229				 KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 230}
 231
 232static struct kvm_cpuid_entry2 *kvm_find_kvm_cpuid_features(struct kvm_vcpu *vcpu)
 233{
 234	return __kvm_find_kvm_cpuid_features(vcpu, vcpu->arch.cpuid_entries,
 235					     vcpu->arch.cpuid_nent);
 236}
 237
 238void kvm_update_pv_runtime(struct kvm_vcpu *vcpu)
 239{
 240	struct kvm_cpuid_entry2 *best = kvm_find_kvm_cpuid_features(vcpu);
 241
 242	/*
 243	 * save the feature bitmap to avoid cpuid lookup for every PV
 244	 * operation
 245	 */
 246	if (best)
 247		vcpu->arch.pv_cpuid.features = best->eax;
 248}
 249
 250/*
 251 * Calculate guest's supported XCR0 taking into account guest CPUID data and
 252 * KVM's supported XCR0 (comprised of host's XCR0 and KVM_SUPPORTED_XCR0).
 253 */
 254static u64 cpuid_get_supported_xcr0(struct kvm_cpuid_entry2 *entries, int nent)
 255{
 256	struct kvm_cpuid_entry2 *best;
 257
 258	best = cpuid_entry2_find(entries, nent, 0xd, 0);
 259	if (!best)
 260		return 0;
 261
 262	return (best->eax | ((u64)best->edx << 32)) & kvm_caps.supported_xcr0;
 263}
 264
 265static void __kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *entries,
 266				       int nent)
 267{
 268	struct kvm_cpuid_entry2 *best;
 269
 270	best = cpuid_entry2_find(entries, nent, 1, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 271	if (best) {
 272		/* Update OSXSAVE bit */
 273		if (boot_cpu_has(X86_FEATURE_XSAVE))
 274			cpuid_entry_change(best, X86_FEATURE_OSXSAVE,
 275					   kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE));
 276
 277		cpuid_entry_change(best, X86_FEATURE_APIC,
 278			   vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
 279	}
 280
 281	best = cpuid_entry2_find(entries, nent, 7, 0);
 282	if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7)
 283		cpuid_entry_change(best, X86_FEATURE_OSPKE,
 284				   kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE));
 285
 286	best = cpuid_entry2_find(entries, nent, 0xD, 0);
 287	if (best)
 288		best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
 289
 290	best = cpuid_entry2_find(entries, nent, 0xD, 1);
 291	if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) ||
 292		     cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
 293		best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
 294
 295	best = __kvm_find_kvm_cpuid_features(vcpu, entries, nent);
 296	if (kvm_hlt_in_guest(vcpu->kvm) && best &&
 297		(best->eax & (1 << KVM_FEATURE_PV_UNHALT)))
 298		best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
 299
 300	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) {
 301		best = cpuid_entry2_find(entries, nent, 0x1, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 302		if (best)
 303			cpuid_entry_change(best, X86_FEATURE_MWAIT,
 304					   vcpu->arch.ia32_misc_enable_msr &
 305					   MSR_IA32_MISC_ENABLE_MWAIT);
 306	}
 307}
 308
 309void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
 310{
 311	__kvm_update_cpuid_runtime(vcpu, vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent);
 312}
 313EXPORT_SYMBOL_GPL(kvm_update_cpuid_runtime);
 314
 315static bool kvm_cpuid_has_hyperv(struct kvm_cpuid_entry2 *entries, int nent)
 316{
 317#ifdef CONFIG_KVM_HYPERV
 318	struct kvm_cpuid_entry2 *entry;
 319
 320	entry = cpuid_entry2_find(entries, nent, HYPERV_CPUID_INTERFACE,
 321				  KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 322	return entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX;
 323#else
 324	return false;
 325#endif
 326}
 327
 328static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 329{
 330	struct kvm_lapic *apic = vcpu->arch.apic;
 331	struct kvm_cpuid_entry2 *best;
 332	bool allow_gbpages;
 333
 334	BUILD_BUG_ON(KVM_NR_GOVERNED_FEATURES > KVM_MAX_NR_GOVERNED_FEATURES);
 335	bitmap_zero(vcpu->arch.governed_features.enabled,
 336		    KVM_MAX_NR_GOVERNED_FEATURES);
 337
 338	/*
 339	 * If TDP is enabled, let the guest use GBPAGES if they're supported in
 340	 * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
 341	 * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
 342	 * walk for performance and complexity reasons.  Not to mention KVM
 343	 * _can't_ solve the problem because GVA->GPA walks aren't visible to
 344	 * KVM once a TDP translation is installed.  Mimic hardware behavior so
 345	 * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
 346	 * If TDP is disabled, honor *only* guest CPUID as KVM has full control
 347	 * and can install smaller shadow pages if the host lacks 1GiB support.
 348	 */
 349	allow_gbpages = tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
 350				      guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
 351	if (allow_gbpages)
 352		kvm_governed_feature_set(vcpu, X86_FEATURE_GBPAGES);
 353
 354	best = kvm_find_cpuid_entry(vcpu, 1);
 355	if (best && apic) {
 356		if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
 357			apic->lapic_timer.timer_mode_mask = 3 << 17;
 358		else
 359			apic->lapic_timer.timer_mode_mask = 1 << 17;
 360
 361		kvm_apic_set_version(vcpu);
 362	}
 363
 364	vcpu->arch.guest_supported_xcr0 =
 365		cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent);
 366
 367	kvm_update_pv_runtime(vcpu);
 
 
 368
 369	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
 370	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
 371
 372	kvm_pmu_refresh(vcpu);
 373	vcpu->arch.cr4_guest_rsvd_bits =
 374	    __cr4_reserved_bits(guest_cpuid_has, vcpu);
 
 
 375
 376	kvm_hv_set_cpuid(vcpu, kvm_cpuid_has_hyperv(vcpu->arch.cpuid_entries,
 377						    vcpu->arch.cpuid_nent));
 
 
 378
 379	/* Invoke the vendor callback only after the above state is updated. */
 380	static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu);
 
 
 381
 382	/*
 383	 * Except for the MMU, which needs to do its thing any vendor specific
 384	 * adjustments to the reserved GPA bits.
 385	 */
 386	kvm_mmu_after_set_cpuid(vcpu);
 
 
 
 
 
 
 
 387}
 388
 389int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
 390{
 391	struct kvm_cpuid_entry2 *best;
 392
 393	best = kvm_find_cpuid_entry(vcpu, 0x80000000);
 394	if (!best || best->eax < 0x80000008)
 395		goto not_found;
 396	best = kvm_find_cpuid_entry(vcpu, 0x80000008);
 397	if (best)
 398		return best->eax & 0xff;
 399not_found:
 400	return 36;
 401}
 402
 403/*
 404 * This "raw" version returns the reserved GPA bits without any adjustments for
 405 * encryption technologies that usurp bits.  The raw mask should be used if and
 406 * only if hardware does _not_ strip the usurped bits, e.g. in virtual MTRRs.
 407 */
 408u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
 409{
 410	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
 411}
 412
 413static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
 414                        int nent)
 415{
 416	int r;
 417
 418	__kvm_update_cpuid_runtime(vcpu, e2, nent);
 419
 420	/*
 421	 * KVM does not correctly handle changing guest CPUID after KVM_RUN, as
 422	 * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't
 423	 * tracked in kvm_mmu_page_role.  As a result, KVM may miss guest page
 424	 * faults due to reusing SPs/SPTEs. In practice no sane VMM mucks with
 425	 * the core vCPU model on the fly. It would've been better to forbid any
 426	 * KVM_SET_CPUID{,2} calls after KVM_RUN altogether but unfortunately
 427	 * some VMMs (e.g. QEMU) reuse vCPU fds for CPU hotplug/unplug and do
 428	 * KVM_SET_CPUID{,2} again. To support this legacy behavior, check
 429	 * whether the supplied CPUID data is equal to what's already set.
 430	 */
 431	if (kvm_vcpu_has_run(vcpu)) {
 432		r = kvm_cpuid_check_equal(vcpu, e2, nent);
 433		if (r)
 434			return r;
 435
 436		kvfree(e2);
 437		return 0;
 438	}
 439
 440#ifdef CONFIG_KVM_HYPERV
 441	if (kvm_cpuid_has_hyperv(e2, nent)) {
 442		r = kvm_hv_vcpu_init(vcpu);
 443		if (r)
 444			return r;
 445	}
 446#endif
 447
 448	r = kvm_check_cpuid(vcpu, e2, nent);
 449	if (r)
 450		return r;
 451
 452	kvfree(vcpu->arch.cpuid_entries);
 453	vcpu->arch.cpuid_entries = e2;
 454	vcpu->arch.cpuid_nent = nent;
 455
 456	vcpu->arch.kvm_cpuid = kvm_get_hypervisor_cpuid(vcpu, KVM_SIGNATURE);
 457#ifdef CONFIG_KVM_XEN
 458	vcpu->arch.xen.cpuid = kvm_get_hypervisor_cpuid(vcpu, XEN_SIGNATURE);
 459#endif
 460	kvm_vcpu_after_set_cpuid(vcpu);
 461
 462	return 0;
 463}
 464
 465/* when an old userspace process fills a new kernel module */
 466int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
 467			     struct kvm_cpuid *cpuid,
 468			     struct kvm_cpuid_entry __user *entries)
 469{
 470	int r, i;
 471	struct kvm_cpuid_entry *e = NULL;
 472	struct kvm_cpuid_entry2 *e2 = NULL;
 473
 
 474	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 475		return -E2BIG;
 476
 477	if (cpuid->nent) {
 478		e = vmemdup_array_user(entries, cpuid->nent, sizeof(*e));
 479		if (IS_ERR(e))
 480			return PTR_ERR(e);
 481
 482		e2 = kvmalloc_array(cpuid->nent, sizeof(*e2), GFP_KERNEL_ACCOUNT);
 483		if (!e2) {
 484			r = -ENOMEM;
 485			goto out_free_cpuid;
 486		}
 487	}
 488	for (i = 0; i < cpuid->nent; i++) {
 489		e2[i].function = e[i].function;
 490		e2[i].eax = e[i].eax;
 491		e2[i].ebx = e[i].ebx;
 492		e2[i].ecx = e[i].ecx;
 493		e2[i].edx = e[i].edx;
 494		e2[i].index = 0;
 495		e2[i].flags = 0;
 496		e2[i].padding[0] = 0;
 497		e2[i].padding[1] = 0;
 498		e2[i].padding[2] = 0;
 
 
 
 
 
 
 
 499	}
 500
 501	r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
 502	if (r)
 503		kvfree(e2);
 504
 505out_free_cpuid:
 506	kvfree(e);
 507
 
 
 508	return r;
 509}
 510
 511int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
 512			      struct kvm_cpuid2 *cpuid,
 513			      struct kvm_cpuid_entry2 __user *entries)
 514{
 515	struct kvm_cpuid_entry2 *e2 = NULL;
 516	int r;
 517
 
 518	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 519		return -E2BIG;
 520
 521	if (cpuid->nent) {
 522		e2 = vmemdup_array_user(entries, cpuid->nent, sizeof(*e2));
 523		if (IS_ERR(e2))
 524			return PTR_ERR(e2);
 
 
 
 
 525	}
 526
 527	r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
 528	if (r)
 529		kvfree(e2);
 530
 531	return r;
 532}
 533
 534int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
 535			      struct kvm_cpuid2 *cpuid,
 536			      struct kvm_cpuid_entry2 __user *entries)
 537{
 538	if (cpuid->nent < vcpu->arch.cpuid_nent)
 539		return -E2BIG;
 540
 541	if (copy_to_user(entries, vcpu->arch.cpuid_entries,
 
 
 
 
 542			 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
 543		return -EFAULT;
 
 544
 
 545	cpuid->nent = vcpu->arch.cpuid_nent;
 546	return 0;
 547}
 548
 549/* Mask kvm_cpu_caps for @leaf with the raw CPUID capabilities of this CPU. */
 550static __always_inline void __kvm_cpu_cap_mask(unsigned int leaf)
 551{
 552	const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);
 553	struct kvm_cpuid_entry2 entry;
 554
 555	reverse_cpuid_check(leaf);
 
 556
 557	cpuid_count(cpuid.function, cpuid.index,
 558		    &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
 559
 560	kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg);
 561}
 562
 563static __always_inline
 564void kvm_cpu_cap_init_kvm_defined(enum kvm_only_cpuid_leafs leaf, u32 mask)
 565{
 566	/* Use kvm_cpu_cap_mask for leafs that aren't KVM-only. */
 567	BUILD_BUG_ON(leaf < NCAPINTS);
 568
 569	kvm_cpu_caps[leaf] = mask;
 570
 571	__kvm_cpu_cap_mask(leaf);
 572}
 573
 574static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask)
 575{
 576	/* Use kvm_cpu_cap_init_kvm_defined for KVM-only leafs. */
 577	BUILD_BUG_ON(leaf >= NCAPINTS);
 578
 579	kvm_cpu_caps[leaf] &= mask;
 580
 581	__kvm_cpu_cap_mask(leaf);
 582}
 583
 584void kvm_set_cpu_caps(void)
 585{
 
 586#ifdef CONFIG_X86_64
 587	unsigned int f_gbpages = F(GBPAGES);
 588	unsigned int f_lm = F(LM);
 589	unsigned int f_xfd = F(XFD);
 590#else
 591	unsigned int f_gbpages = 0;
 592	unsigned int f_lm = 0;
 593	unsigned int f_xfd = 0;
 594#endif
 595	memset(kvm_cpu_caps, 0, sizeof(kvm_cpu_caps));
 596
 597	BUILD_BUG_ON(sizeof(kvm_cpu_caps) - (NKVMCAPINTS * sizeof(*kvm_cpu_caps)) >
 598		     sizeof(boot_cpu_data.x86_capability));
 599
 600	memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability,
 601	       sizeof(kvm_cpu_caps) - (NKVMCAPINTS * sizeof(*kvm_cpu_caps)));
 602
 603	kvm_cpu_cap_mask(CPUID_1_ECX,
 604		/*
 605		 * NOTE: MONITOR (and MWAIT) are emulated as NOP, but *not*
 606		 * advertised to guests via CPUID!
 607		 */
 608		F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
 609		0 /* DS-CPL, VMX, SMX, EST */ |
 610		0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
 611		F(FMA) | F(CX16) | 0 /* xTPR Update */ | F(PDCM) |
 612		F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) |
 613		F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
 614		0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
 615		F(F16C) | F(RDRAND)
 616	);
 617	/* KVM emulates x2apic in software irrespective of host support. */
 618	kvm_cpu_cap_set(X86_FEATURE_X2APIC);
 619
 620	kvm_cpu_cap_mask(CPUID_1_EDX,
 621		F(FPU) | F(VME) | F(DE) | F(PSE) |
 622		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
 623		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
 624		F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
 625		F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLUSH) |
 626		0 /* Reserved, DS, ACPI */ | F(MMX) |
 627		F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
 628		0 /* HTT, TM, Reserved, PBE */
 629	);
 630
 631	kvm_cpu_cap_mask(CPUID_7_0_EBX,
 632		F(FSGSBASE) | F(SGX) | F(BMI1) | F(HLE) | F(AVX2) |
 633		F(FDP_EXCPTN_ONLY) | F(SMEP) | F(BMI2) | F(ERMS) | F(INVPCID) |
 634		F(RTM) | F(ZERO_FCS_FDS) | 0 /*MPX*/ | F(AVX512F) |
 635		F(AVX512DQ) | F(RDSEED) | F(ADX) | F(SMAP) | F(AVX512IFMA) |
 636		F(CLFLUSHOPT) | F(CLWB) | 0 /*INTEL_PT*/ | F(AVX512PF) |
 637		F(AVX512ER) | F(AVX512CD) | F(SHA_NI) | F(AVX512BW) |
 638		F(AVX512VL));
 639
 640	kvm_cpu_cap_mask(CPUID_7_ECX,
 641		F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(RDPID) |
 642		F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
 643		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
 644		F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/ |
 645		F(SGX_LC) | F(BUS_LOCK_DETECT)
 646	);
 647	/* Set LA57 based on hardware capability. */
 648	if (cpuid_ecx(7) & F(LA57))
 649		kvm_cpu_cap_set(X86_FEATURE_LA57);
 650
 651	/*
 652	 * PKU not yet implemented for shadow paging and requires OSPKE
 653	 * to be set on the host. Clear it if that is not the case
 654	 */
 655	if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
 656		kvm_cpu_cap_clear(X86_FEATURE_PKU);
 657
 658	kvm_cpu_cap_mask(CPUID_7_EDX,
 659		F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
 660		F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
 661		F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) |
 662		F(SERIALIZE) | F(TSXLDTRK) | F(AVX512_FP16) |
 663		F(AMX_TILE) | F(AMX_INT8) | F(AMX_BF16) | F(FLUSH_L1D)
 664	);
 665
 666	/* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */
 667	kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST);
 668	kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES);
 669
 670	if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
 671		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
 672	if (boot_cpu_has(X86_FEATURE_STIBP))
 673		kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
 674	if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
 675		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
 676
 677	kvm_cpu_cap_mask(CPUID_7_1_EAX,
 678		F(AVX_VNNI) | F(AVX512_BF16) | F(CMPCCXADD) |
 679		F(FZRM) | F(FSRS) | F(FSRC) |
 680		F(AMX_FP16) | F(AVX_IFMA) | F(LAM)
 681	);
 682
 683	kvm_cpu_cap_init_kvm_defined(CPUID_7_1_EDX,
 684		F(AVX_VNNI_INT8) | F(AVX_NE_CONVERT) | F(PREFETCHITI) |
 685		F(AMX_COMPLEX)
 686	);
 687
 688	kvm_cpu_cap_init_kvm_defined(CPUID_7_2_EDX,
 689		F(INTEL_PSFD) | F(IPRED_CTRL) | F(RRSBA_CTRL) | F(DDPD_U) |
 690		F(BHI_CTRL) | F(MCDT_NO)
 691	);
 692
 693	kvm_cpu_cap_mask(CPUID_D_1_EAX,
 694		F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) | f_xfd
 695	);
 696
 697	kvm_cpu_cap_init_kvm_defined(CPUID_12_EAX,
 698		SF(SGX1) | SF(SGX2) | SF(SGX_EDECCSSA)
 699	);
 700
 701	kvm_cpu_cap_mask(CPUID_8000_0001_ECX,
 702		F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
 703		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
 704		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
 705		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
 706		F(TOPOEXT) | 0 /* PERFCTR_CORE */
 707	);
 708
 709	kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
 710		F(FPU) | F(VME) | F(DE) | F(PSE) |
 711		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
 712		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
 713		F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
 714		F(PAT) | F(PSE36) | 0 /* Reserved */ |
 715		F(NX) | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
 716		F(FXSR) | F(FXSR_OPT) | f_gbpages | F(RDTSCP) |
 717		0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW)
 718	);
 719
 720	if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
 721		kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
 722
 723	kvm_cpu_cap_init_kvm_defined(CPUID_8000_0007_EDX,
 724		SF(CONSTANT_TSC)
 725	);
 726
 727	kvm_cpu_cap_mask(CPUID_8000_0008_EBX,
 728		F(CLZERO) | F(XSAVEERPTR) |
 729		F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) |
 730		F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON) |
 731		F(AMD_PSFD)
 732	);
 733
 734	/*
 735	 * AMD has separate bits for each SPEC_CTRL bit.
 736	 * arch/x86/kernel/cpu/bugs.c is kind enough to
 737	 * record that in cpufeatures so use them.
 738	 */
 739	if (boot_cpu_has(X86_FEATURE_IBPB))
 740		kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
 741	if (boot_cpu_has(X86_FEATURE_IBRS))
 742		kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
 743	if (boot_cpu_has(X86_FEATURE_STIBP))
 744		kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
 745	if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
 746		kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
 747	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 748		kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
 749	/*
 750	 * The preference is to use SPEC CTRL MSR instead of the
 751	 * VIRT_SPEC MSR.
 752	 */
 753	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 754	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
 755		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 756
 757	/*
 758	 * Hide all SVM features by default, SVM will set the cap bits for
 759	 * features it emulates and/or exposes for L1.
 760	 */
 761	kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0);
 762
 763	kvm_cpu_cap_mask(CPUID_8000_001F_EAX,
 764		0 /* SME */ | F(SEV) | 0 /* VM_PAGE_FLUSH */ | F(SEV_ES) |
 765		F(SME_COHERENT));
 766
 767	kvm_cpu_cap_mask(CPUID_8000_0021_EAX,
 768		F(NO_NESTED_DATA_BP) | F(LFENCE_RDTSC) | 0 /* SmmPgCfgLock */ |
 769		F(NULL_SEL_CLR_BASE) | F(AUTOIBRS) | 0 /* PrefetchCtlMsr */ |
 770		F(WRMSR_XX_BASE_NS)
 771	);
 772
 773	kvm_cpu_cap_check_and_set(X86_FEATURE_SBPB);
 774	kvm_cpu_cap_check_and_set(X86_FEATURE_IBPB_BRTYPE);
 775	kvm_cpu_cap_check_and_set(X86_FEATURE_SRSO_NO);
 776
 777	kvm_cpu_cap_init_kvm_defined(CPUID_8000_0022_EAX,
 778		F(PERFMON_V2)
 779	);
 780
 781	/*
 782	 * Synthesize "LFENCE is serializing" into the AMD-defined entry in
 783	 * KVM's supported CPUID if the feature is reported as supported by the
 784	 * kernel.  LFENCE_RDTSC was a Linux-defined synthetic feature long
 785	 * before AMD joined the bandwagon, e.g. LFENCE is serializing on most
 786	 * CPUs that support SSE2.  On CPUs that don't support AMD's leaf,
 787	 * kvm_cpu_cap_mask() will unfortunately drop the flag due to ANDing
 788	 * the mask with the raw host CPUID, and reporting support in AMD's
 789	 * leaf can make it easier for userspace to detect the feature.
 790	 */
 791	if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
 792		kvm_cpu_cap_set(X86_FEATURE_LFENCE_RDTSC);
 793	if (!static_cpu_has_bug(X86_BUG_NULL_SEG))
 794		kvm_cpu_cap_set(X86_FEATURE_NULL_SEL_CLR_BASE);
 795	kvm_cpu_cap_set(X86_FEATURE_NO_SMM_CTL_MSR);
 796
 797	kvm_cpu_cap_mask(CPUID_C000_0001_EDX,
 798		F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) |
 799		F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) |
 800		F(PMM) | F(PMM_EN)
 801	);
 802
 803	/*
 804	 * Hide RDTSCP and RDPID if either feature is reported as supported but
 805	 * probing MSR_TSC_AUX failed.  This is purely a sanity check and
 806	 * should never happen, but the guest will likely crash if RDTSCP or
 807	 * RDPID is misreported, and KVM has botched MSR_TSC_AUX emulation in
 808	 * the past.  For example, the sanity check may fire if this instance of
 809	 * KVM is running as L1 on top of an older, broken KVM.
 810	 */
 811	if (WARN_ON((kvm_cpu_cap_has(X86_FEATURE_RDTSCP) ||
 812		     kvm_cpu_cap_has(X86_FEATURE_RDPID)) &&
 813		     !kvm_is_supported_user_return_msr(MSR_TSC_AUX))) {
 814		kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
 815		kvm_cpu_cap_clear(X86_FEATURE_RDPID);
 816	}
 817}
 818EXPORT_SYMBOL_GPL(kvm_set_cpu_caps);
 819
 820struct kvm_cpuid_array {
 821	struct kvm_cpuid_entry2 *entries;
 822	int maxnent;
 823	int nent;
 824};
 825
 826static struct kvm_cpuid_entry2 *get_next_cpuid(struct kvm_cpuid_array *array)
 827{
 828	if (array->nent >= array->maxnent)
 829		return NULL;
 830
 831	return &array->entries[array->nent++];
 832}
 833
 834static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
 835					      u32 function, u32 index)
 836{
 837	struct kvm_cpuid_entry2 *entry = get_next_cpuid(array);
 838
 839	if (!entry)
 840		return NULL;
 841
 842	memset(entry, 0, sizeof(*entry));
 
 843	entry->function = function;
 844	entry->index = index;
 845	switch (function & 0xC0000000) {
 846	case 0x40000000:
 847		/* Hypervisor leaves are always synthesized by __do_cpuid_func.  */
 848		return entry;
 849
 850	case 0x80000000:
 851		/*
 852		 * 0x80000021 is sometimes synthesized by __do_cpuid_func, which
 853		 * would result in out-of-bounds calls to do_host_cpuid.
 854		 */
 855		{
 856			static int max_cpuid_80000000;
 857			if (!READ_ONCE(max_cpuid_80000000))
 858				WRITE_ONCE(max_cpuid_80000000, cpuid_eax(0x80000000));
 859			if (function > READ_ONCE(max_cpuid_80000000))
 860				return entry;
 861		}
 862		break;
 863
 864	default:
 865		break;
 866	}
 867
 868	cpuid_count(entry->function, entry->index,
 869		    &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
 870
 871	if (cpuid_function_is_indexed(function))
 
 
 
 
 
 
 
 
 
 
 
 
 872		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 
 
 873
 874	return entry;
 875}
 876
 877static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
 878{
 879	struct kvm_cpuid_entry2 *entry;
 880
 881	if (array->nent >= array->maxnent)
 882		return -E2BIG;
 883
 884	entry = &array->entries[array->nent];
 885	entry->function = func;
 886	entry->index = 0;
 887	entry->flags = 0;
 888
 889	switch (func) {
 890	case 0:
 891		entry->eax = 7;
 892		++array->nent;
 893		break;
 894	case 1:
 895		entry->ecx = F(MOVBE);
 896		++array->nent;
 897		break;
 898	case 7:
 899		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 900		entry->eax = 0;
 901		if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
 902			entry->ecx = F(RDPID);
 903		++array->nent;
 904		break;
 905	default:
 906		break;
 907	}
 908
 909	return 0;
 910}
 911
 912static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 913{
 914	struct kvm_cpuid_entry2 *entry;
 915	int r, i, max_idx;
 916
 917	/* all calls to cpuid_count() should be made on the same cpu */
 918	get_cpu();
 919
 920	r = -E2BIG;
 921
 922	entry = do_host_cpuid(array, function, 0);
 923	if (!entry)
 924		goto out;
 925
 926	switch (function) {
 927	case 0:
 928		/* Limited to the highest leaf implemented in KVM. */
 929		entry->eax = min(entry->eax, 0x1fU);
 930		break;
 931	case 1:
 932		cpuid_entry_override(entry, CPUID_1_EDX);
 933		cpuid_entry_override(entry, CPUID_1_ECX);
 934		break;
 935	case 2:
 936		/*
 937		 * On ancient CPUs, function 2 entries are STATEFUL.  That is,
 938		 * CPUID(function=2, index=0) may return different results each
 939		 * time, with the least-significant byte in EAX enumerating the
 940		 * number of times software should do CPUID(2, 0).
 941		 *
 942		 * Modern CPUs, i.e. every CPU KVM has *ever* run on are less
 943		 * idiotic.  Intel's SDM states that EAX & 0xff "will always
 944		 * return 01H. Software should ignore this value and not
 945		 * interpret it as an informational descriptor", while AMD's
 946		 * APM states that CPUID(2) is reserved.
 947		 *
 948		 * WARN if a frankenstein CPU that supports virtualization and
 949		 * a stateful CPUID.0x2 is encountered.
 950		 */
 951		WARN_ON_ONCE((entry->eax & 0xff) > 1);
 952		break;
 953	/* functions 4 and 0x8000001d have additional index. */
 954	case 4:
 955	case 0x8000001d:
 956		/*
 957		 * Read entries until the cache type in the previous entry is
 958		 * zero, i.e. indicates an invalid entry.
 959		 */
 960		for (i = 1; entry->eax & 0x1f; ++i) {
 961			entry = do_host_cpuid(array, function, i);
 962			if (!entry)
 963				goto out;
 964		}
 965		break;
 966	case 6: /* Thermal management */
 967		entry->eax = 0x4; /* allow ARAT */
 968		entry->ebx = 0;
 969		entry->ecx = 0;
 970		entry->edx = 0;
 971		break;
 972	/* function 7 has additional index. */
 973	case 7:
 974		max_idx = entry->eax = min(entry->eax, 2u);
 975		cpuid_entry_override(entry, CPUID_7_0_EBX);
 976		cpuid_entry_override(entry, CPUID_7_ECX);
 977		cpuid_entry_override(entry, CPUID_7_EDX);
 978
 979		/* KVM only supports up to 0x7.2, capped above via min(). */
 980		if (max_idx >= 1) {
 981			entry = do_host_cpuid(array, function, 1);
 982			if (!entry)
 983				goto out;
 984
 985			cpuid_entry_override(entry, CPUID_7_1_EAX);
 986			cpuid_entry_override(entry, CPUID_7_1_EDX);
 987			entry->ebx = 0;
 988			entry->ecx = 0;
 989		}
 990		if (max_idx >= 2) {
 991			entry = do_host_cpuid(array, function, 2);
 992			if (!entry)
 993				goto out;
 994
 995			cpuid_entry_override(entry, CPUID_7_2_EDX);
 996			entry->ecx = 0;
 997			entry->ebx = 0;
 998			entry->eax = 0;
 999		}
1000		break;
 
 
1001	case 0xa: { /* Architectural Performance Monitoring */
 
1002		union cpuid10_eax eax;
1003		union cpuid10_edx edx;
1004
1005		if (!enable_pmu || !static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
1006			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1007			break;
1008		}
1009
1010		eax.split.version_id = kvm_pmu_cap.version;
1011		eax.split.num_counters = kvm_pmu_cap.num_counters_gp;
1012		eax.split.bit_width = kvm_pmu_cap.bit_width_gp;
1013		eax.split.mask_length = kvm_pmu_cap.events_mask_len;
1014		edx.split.num_counters_fixed = kvm_pmu_cap.num_counters_fixed;
1015		edx.split.bit_width_fixed = kvm_pmu_cap.bit_width_fixed;
1016
1017		if (kvm_pmu_cap.version)
1018			edx.split.anythread_deprecated = 1;
1019		edx.split.reserved1 = 0;
1020		edx.split.reserved2 = 0;
 
 
 
 
1021
1022		entry->eax = eax.full;
1023		entry->ebx = kvm_pmu_cap.events_mask;
1024		entry->ecx = 0;
1025		entry->edx = edx.full;
1026		break;
1027	}
 
 
 
 
1028	case 0x1f:
1029	case 0xb:
1030		/*
1031		 * No topology; a valid topology is indicated by the presence
1032		 * of subleaf 1.
 
1033		 */
1034		entry->eax = entry->ebx = entry->ecx = 0;
 
 
 
 
1035		break;
1036	case 0xd: {
1037		u64 permitted_xcr0 = kvm_get_filtered_xcr0();
1038		u64 permitted_xss = kvm_caps.supported_xss;
1039
1040		entry->eax &= permitted_xcr0;
1041		entry->ebx = xstate_required_size(permitted_xcr0, false);
1042		entry->ecx = entry->ebx;
1043		entry->edx &= permitted_xcr0 >> 32;
1044		if (!permitted_xcr0)
1045			break;
1046
1047		entry = do_host_cpuid(array, function, 1);
1048		if (!entry)
1049			goto out;
1050
1051		cpuid_entry_override(entry, CPUID_D_1_EAX);
1052		if (entry->eax & (F(XSAVES)|F(XSAVEC)))
1053			entry->ebx = xstate_required_size(permitted_xcr0 | permitted_xss,
1054							  true);
1055		else {
1056			WARN_ON_ONCE(permitted_xss != 0);
1057			entry->ebx = 0;
1058		}
1059		entry->ecx &= permitted_xss;
1060		entry->edx &= permitted_xss >> 32;
1061
1062		for (i = 2; i < 64; ++i) {
1063			bool s_state;
1064			if (permitted_xcr0 & BIT_ULL(i))
1065				s_state = false;
1066			else if (permitted_xss & BIT_ULL(i))
1067				s_state = true;
1068			else
1069				continue;
1070
1071			entry = do_host_cpuid(array, function, i);
1072			if (!entry)
1073				goto out;
1074
1075			/*
1076			 * The supported check above should have filtered out
1077			 * invalid sub-leafs.  Only valid sub-leafs should
1078			 * reach this point, and they should have a non-zero
1079			 * save state size.  Furthermore, check whether the
1080			 * processor agrees with permitted_xcr0/permitted_xss
1081			 * on whether this is an XCR0- or IA32_XSS-managed area.
1082			 */
1083			if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) {
1084				--array->nent;
1085				continue;
1086			}
1087
1088			if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
1089				entry->ecx &= ~BIT_ULL(2);
1090			entry->edx = 0;
1091		}
1092		break;
1093	}
1094	case 0x12:
1095		/* Intel SGX */
1096		if (!kvm_cpu_cap_has(X86_FEATURE_SGX)) {
1097			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1098			break;
1099		}
1100
1101		/*
1102		 * Index 0: Sub-features, MISCSELECT (a.k.a extended features)
1103		 * and max enclave sizes.   The SGX sub-features and MISCSELECT
1104		 * are restricted by kernel and KVM capabilities (like most
1105		 * feature flags), while enclave size is unrestricted.
1106		 */
1107		cpuid_entry_override(entry, CPUID_12_EAX);
1108		entry->ebx &= SGX_MISC_EXINFO;
1109
1110		entry = do_host_cpuid(array, function, 1);
1111		if (!entry)
1112			goto out;
1113
1114		/*
1115		 * Index 1: SECS.ATTRIBUTES.  ATTRIBUTES are restricted a la
1116		 * feature flags.  Advertise all supported flags, including
1117		 * privileged attributes that require explicit opt-in from
1118		 * userspace.  ATTRIBUTES.XFRM is not adjusted as userspace is
1119		 * expected to derive it from supported XCR0.
1120		 */
1121		entry->eax &= SGX_ATTR_PRIV_MASK | SGX_ATTR_UNPRIV_MASK;
1122		entry->ebx &= 0;
1123		break;
1124	/* Intel PT */
1125	case 0x14:
1126		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
1127			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1128			break;
1129		}
1130
1131		for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1132			if (!do_host_cpuid(array, function, i))
1133				goto out;
1134		}
1135		break;
1136	/* Intel AMX TILE */
1137	case 0x1d:
1138		if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1139			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1140			break;
1141		}
1142
1143		for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1144			if (!do_host_cpuid(array, function, i))
1145				goto out;
1146		}
1147		break;
1148	case 0x1e: /* TMUL information */
1149		if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1150			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1151			break;
1152		}
1153		break;
1154	case KVM_CPUID_SIGNATURE: {
1155		const u32 *sigptr = (const u32 *)KVM_SIGNATURE;
 
1156		entry->eax = KVM_CPUID_FEATURES;
1157		entry->ebx = sigptr[0];
1158		entry->ecx = sigptr[1];
1159		entry->edx = sigptr[2];
1160		break;
1161	}
1162	case KVM_CPUID_FEATURES:
1163		entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
1164			     (1 << KVM_FEATURE_NOP_IO_DELAY) |
1165			     (1 << KVM_FEATURE_CLOCKSOURCE2) |
1166			     (1 << KVM_FEATURE_ASYNC_PF) |
1167			     (1 << KVM_FEATURE_PV_EOI) |
1168			     (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
1169			     (1 << KVM_FEATURE_PV_UNHALT) |
1170			     (1 << KVM_FEATURE_PV_TLB_FLUSH) |
1171			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
1172			     (1 << KVM_FEATURE_PV_SEND_IPI) |
1173			     (1 << KVM_FEATURE_POLL_CONTROL) |
1174			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
1175			     (1 << KVM_FEATURE_ASYNC_PF_INT);
1176
1177		if (sched_info_on())
1178			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
1179
1180		entry->ebx = 0;
1181		entry->ecx = 0;
1182		entry->edx = 0;
1183		break;
1184	case 0x80000000:
1185		entry->eax = min(entry->eax, 0x80000022);
1186		/*
1187		 * Serializing LFENCE is reported in a multitude of ways, and
1188		 * NullSegClearsBase is not reported in CPUID on Zen2; help
1189		 * userspace by providing the CPUID leaf ourselves.
1190		 *
1191		 * However, only do it if the host has CPUID leaf 0x8000001d.
1192		 * QEMU thinks that it can query the host blindly for that
1193		 * CPUID leaf if KVM reports that it supports 0x8000001d or
1194		 * above.  The processor merrily returns values from the
1195		 * highest Intel leaf which QEMU tries to use as the guest's
1196		 * 0x8000001d.  Even worse, this can result in an infinite
1197		 * loop if said highest leaf has no subleaves indexed by ECX.
1198		 */
1199		if (entry->eax >= 0x8000001d &&
1200		    (static_cpu_has(X86_FEATURE_LFENCE_RDTSC)
1201		     || !static_cpu_has_bug(X86_BUG_NULL_SEG)))
1202			entry->eax = max(entry->eax, 0x80000021);
1203		break;
1204	case 0x80000001:
1205		entry->ebx &= ~GENMASK(27, 16);
1206		cpuid_entry_override(entry, CPUID_8000_0001_EDX);
1207		cpuid_entry_override(entry, CPUID_8000_0001_ECX);
1208		break;
1209	case 0x80000005:
1210		/*  Pass host L1 cache and TLB info. */
1211		break;
1212	case 0x80000006:
1213		/* Drop reserved bits, pass host L2 cache and TLB info. */
1214		entry->edx &= ~GENMASK(17, 16);
1215		break;
1216	case 0x80000007: /* Advanced power management */
1217		cpuid_entry_override(entry, CPUID_8000_0007_EDX);
1218
1219		/* mask against host */
1220		entry->edx &= boot_cpu_data.x86_power;
1221		entry->eax = entry->ebx = entry->ecx = 0;
1222		break;
1223	case 0x80000008: {
1224		unsigned g_phys_as = (entry->eax >> 16) & 0xff;
1225		unsigned virt_as = max((entry->eax >> 8) & 0xff, 48U);
1226		unsigned phys_as = entry->eax & 0xff;
1227
1228		/*
1229		 * If TDP (NPT) is disabled use the adjusted host MAXPHYADDR as
1230		 * the guest operates in the same PA space as the host, i.e.
1231		 * reductions in MAXPHYADDR for memory encryption affect shadow
1232		 * paging, too.
1233		 *
1234		 * If TDP is enabled but an explicit guest MAXPHYADDR is not
1235		 * provided, use the raw bare metal MAXPHYADDR as reductions to
1236		 * the HPAs do not affect GPAs.
1237		 */
1238		if (!tdp_enabled)
1239			g_phys_as = boot_cpu_data.x86_phys_bits;
1240		else if (!g_phys_as)
1241			g_phys_as = phys_as;
1242
1243		entry->eax = g_phys_as | (virt_as << 8);
1244		entry->ecx &= ~(GENMASK(31, 16) | GENMASK(11, 8));
1245		entry->edx = 0;
1246		cpuid_entry_override(entry, CPUID_8000_0008_EBX);
1247		break;
1248	}
1249	case 0x8000000A:
1250		if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
1251			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1252			break;
1253		}
1254		entry->eax = 1; /* SVM revision 1 */
1255		entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
1256				   ASID emulation to nested SVM */
1257		entry->ecx = 0; /* Reserved */
1258		cpuid_entry_override(entry, CPUID_8000_000A_EDX);
1259		break;
1260	case 0x80000019:
1261		entry->ecx = entry->edx = 0;
1262		break;
1263	case 0x8000001a:
1264		entry->eax &= GENMASK(2, 0);
1265		entry->ebx = entry->ecx = entry->edx = 0;
1266		break;
1267	case 0x8000001e:
1268		/* Do not return host topology information.  */
1269		entry->eax = entry->ebx = entry->ecx = 0;
1270		entry->edx = 0; /* reserved */
1271		break;
 
1272	case 0x8000001F:
1273		if (!kvm_cpu_cap_has(X86_FEATURE_SEV)) {
1274			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1275		} else {
1276			cpuid_entry_override(entry, CPUID_8000_001F_EAX);
1277			/* Clear NumVMPL since KVM does not support VMPL.  */
1278			entry->ebx &= ~GENMASK(31, 12);
1279			/*
1280			 * Enumerate '0' for "PA bits reduction", the adjusted
1281			 * MAXPHYADDR is enumerated directly (see 0x80000008).
1282			 */
1283			entry->ebx &= ~GENMASK(11, 6);
1284		}
1285		break;
1286	case 0x80000020:
1287		entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1288		break;
1289	case 0x80000021:
1290		entry->ebx = entry->ecx = entry->edx = 0;
1291		cpuid_entry_override(entry, CPUID_8000_0021_EAX);
1292		break;
1293	/* AMD Extended Performance Monitoring and Debug */
1294	case 0x80000022: {
1295		union cpuid_0x80000022_ebx ebx;
1296
1297		entry->ecx = entry->edx = 0;
1298		if (!enable_pmu || !kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) {
1299			entry->eax = entry->ebx;
1300			break;
1301		}
1302
1303		cpuid_entry_override(entry, CPUID_8000_0022_EAX);
1304
1305		if (kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2))
1306			ebx.split.num_core_pmc = kvm_pmu_cap.num_counters_gp;
1307		else if (kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
1308			ebx.split.num_core_pmc = AMD64_NUM_COUNTERS_CORE;
1309		else
1310			ebx.split.num_core_pmc = AMD64_NUM_COUNTERS;
1311
1312		entry->ebx = ebx.full;
1313		break;
1314	}
1315	/*Add support for Centaur's CPUID instruction*/
1316	case 0xC0000000:
1317		/*Just support up to 0xC0000004 now*/
1318		entry->eax = min(entry->eax, 0xC0000004);
1319		break;
1320	case 0xC0000001:
1321		cpuid_entry_override(entry, CPUID_C000_0001_EDX);
1322		break;
1323	case 3: /* Processor serial number */
1324	case 5: /* MONITOR/MWAIT */
1325	case 0xC0000002:
1326	case 0xC0000003:
1327	case 0xC0000004:
1328	default:
1329		entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1330		break;
1331	}
1332
1333	r = 0;
1334
1335out:
1336	put_cpu();
1337
1338	return r;
1339}
1340
1341static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1342			 unsigned int type)
1343{
1344	if (type == KVM_GET_EMULATED_CPUID)
1345		return __do_cpuid_func_emulated(array, func);
1346
1347	return __do_cpuid_func(array, func);
1348}
1349
1350#define CENTAUR_CPUID_SIGNATURE 0xC0000000
1351
1352static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1353			  unsigned int type)
1354{
1355	u32 limit;
1356	int r;
1357
1358	if (func == CENTAUR_CPUID_SIGNATURE &&
1359	    boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR)
1360		return 0;
1361
1362	r = do_cpuid_func(array, func, type);
1363	if (r)
1364		return r;
1365
1366	limit = array->entries[array->nent - 1].eax;
1367	for (func = func + 1; func <= limit; ++func) {
1368		r = do_cpuid_func(array, func, type);
1369		if (r)
1370			break;
1371	}
1372
1373	return r;
1374}
1375
1376static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
1377				 __u32 num_entries, unsigned int ioctl_type)
1378{
1379	int i;
1380	__u32 pad[3];
1381
1382	if (ioctl_type != KVM_GET_EMULATED_CPUID)
1383		return false;
1384
1385	/*
1386	 * We want to make sure that ->padding is being passed clean from
1387	 * userspace in case we want to use it for something in the future.
1388	 *
1389	 * Sadly, this wasn't enforced for KVM_GET_SUPPORTED_CPUID and so we
1390	 * have to give ourselves satisfied only with the emulated side. /me
1391	 * sheds a tear.
1392	 */
1393	for (i = 0; i < num_entries; i++) {
1394		if (copy_from_user(pad, entries[i].padding, sizeof(pad)))
1395			return true;
1396
1397		if (pad[0] || pad[1] || pad[2])
1398			return true;
1399	}
1400	return false;
1401}
1402
1403int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid,
1404			    struct kvm_cpuid_entry2 __user *entries,
1405			    unsigned int type)
1406{
1407	static const u32 funcs[] = {
1408		0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
1409	};
1410
1411	struct kvm_cpuid_array array = {
1412		.nent = 0,
1413	};
1414	int r, i;
1415
1416	if (cpuid->nent < 1)
1417		return -E2BIG;
1418	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1419		cpuid->nent = KVM_MAX_CPUID_ENTRIES;
1420
1421	if (sanity_check_entries(entries, cpuid->nent, type))
1422		return -EINVAL;
1423
1424	array.entries = kvcalloc(cpuid->nent, sizeof(struct kvm_cpuid_entry2), GFP_KERNEL);
 
1425	if (!array.entries)
1426		return -ENOMEM;
1427
1428	array.maxnent = cpuid->nent;
1429
1430	for (i = 0; i < ARRAY_SIZE(funcs); i++) {
1431		r = get_cpuid_func(&array, funcs[i], type);
1432		if (r)
1433			goto out_free;
1434	}
1435	cpuid->nent = array.nent;
1436
1437	if (copy_to_user(entries, array.entries,
1438			 array.nent * sizeof(struct kvm_cpuid_entry2)))
1439		r = -EFAULT;
1440
1441out_free:
1442	kvfree(array.entries);
1443	return r;
1444}
1445
1446struct kvm_cpuid_entry2 *kvm_find_cpuid_entry_index(struct kvm_vcpu *vcpu,
1447						    u32 function, u32 index)
1448{
1449	return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
1450				 function, index);
1451}
1452EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry_index);
1453
1454struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
1455					      u32 function)
1456{
1457	return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
1458				 function, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
 
 
 
 
 
 
 
 
 
1459}
1460EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
1461
1462/*
1463 * Intel CPUID semantics treats any query for an out-of-range leaf as if the
1464 * highest basic leaf (i.e. CPUID.0H:EAX) were requested.  AMD CPUID semantics
1465 * returns all zeroes for any undefined leaf, whether or not the leaf is in
1466 * range.  Centaur/VIA follows Intel semantics.
1467 *
1468 * A leaf is considered out-of-range if its function is higher than the maximum
1469 * supported leaf of its associated class or if its associated class does not
1470 * exist.
1471 *
1472 * There are three primary classes to be considered, with their respective
1473 * ranges described as "<base> - <top>[,<base2> - <top2>] inclusive.  A primary
1474 * class exists if a guest CPUID entry for its <base> leaf exists.  For a given
1475 * class, CPUID.<base>.EAX contains the max supported leaf for the class.
1476 *
1477 *  - Basic:      0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff
1478 *  - Hypervisor: 0x40000000 - 0x4fffffff
1479 *  - Extended:   0x80000000 - 0xbfffffff
1480 *  - Centaur:    0xc0000000 - 0xcfffffff
1481 *
1482 * The Hypervisor class is further subdivided into sub-classes that each act as
1483 * their own independent class associated with a 0x100 byte range.  E.g. if Qemu
1484 * is advertising support for both HyperV and KVM, the resulting Hypervisor
1485 * CPUID sub-classes are:
1486 *
1487 *  - HyperV:     0x40000000 - 0x400000ff
1488 *  - KVM:        0x40000100 - 0x400001ff
1489 */
1490static struct kvm_cpuid_entry2 *
1491get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index)
1492{
1493	struct kvm_cpuid_entry2 *basic, *class;
1494	u32 function = *fn_ptr;
1495
1496	basic = kvm_find_cpuid_entry(vcpu, 0);
1497	if (!basic)
1498		return NULL;
1499
1500	if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) ||
1501	    is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
1502		return NULL;
1503
1504	if (function >= 0x40000000 && function <= 0x4fffffff)
1505		class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00);
1506	else if (function >= 0xc0000000)
1507		class = kvm_find_cpuid_entry(vcpu, 0xc0000000);
1508	else
1509		class = kvm_find_cpuid_entry(vcpu, function & 0x80000000);
1510
1511	if (class && function <= class->eax)
1512		return NULL;
1513
1514	/*
1515	 * Leaf specific adjustments are also applied when redirecting to the
1516	 * max basic entry, e.g. if the max basic leaf is 0xb but there is no
1517	 * entry for CPUID.0xb.index (see below), then the output value for EDX
1518	 * needs to be pulled from CPUID.0xb.1.
1519	 */
1520	*fn_ptr = basic->eax;
1521
1522	/*
1523	 * The class does not exist or the requested function is out of range;
1524	 * the effective CPUID entry is the max basic leaf.  Note, the index of
1525	 * the original requested leaf is observed!
1526	 */
1527	return kvm_find_cpuid_entry_index(vcpu, basic->eax, index);
1528}
1529
1530bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
1531	       u32 *ecx, u32 *edx, bool exact_only)
1532{
1533	u32 orig_function = *eax, function = *eax, index = *ecx;
1534	struct kvm_cpuid_entry2 *entry;
1535	bool exact, used_max_basic = false;
1536
1537	entry = kvm_find_cpuid_entry_index(vcpu, function, index);
1538	exact = !!entry;
1539
1540	if (!entry && !exact_only) {
1541		entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
1542		used_max_basic = !!entry;
1543	}
1544
1545	if (entry) {
1546		*eax = entry->eax;
1547		*ebx = entry->ebx;
1548		*ecx = entry->ecx;
1549		*edx = entry->edx;
1550		if (function == 7 && index == 0) {
1551			u64 data;
1552		        if (!__kvm_get_msr(vcpu, MSR_IA32_TSX_CTRL, &data, true) &&
1553			    (data & TSX_CTRL_CPUID_CLEAR))
1554				*ebx &= ~(F(RTM) | F(HLE));
1555		} else if (function == 0x80000007) {
1556			if (kvm_hv_invtsc_suppressed(vcpu))
1557				*edx &= ~SF(CONSTANT_TSC);
1558		}
1559	} else {
1560		*eax = *ebx = *ecx = *edx = 0;
1561		/*
1562		 * When leaf 0BH or 1FH is defined, CL is pass-through
1563		 * and EDX is always the x2APIC ID, even for undefined
1564		 * subleaves. Index 1 will exist iff the leaf is
1565		 * implemented, so we pass through CL iff leaf 1
1566		 * exists. EDX can be copied from any existing index.
1567		 */
1568		if (function == 0xb || function == 0x1f) {
1569			entry = kvm_find_cpuid_entry_index(vcpu, function, 1);
1570			if (entry) {
1571				*ecx = index & 0xff;
1572				*edx = entry->edx;
1573			}
1574		}
1575	}
1576	trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact,
1577			used_max_basic);
1578	return exact;
1579}
1580EXPORT_SYMBOL_GPL(kvm_cpuid);
1581
1582int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1583{
1584	u32 eax, ebx, ecx, edx;
1585
1586	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
1587		return 1;
1588
1589	eax = kvm_rax_read(vcpu);
1590	ecx = kvm_rcx_read(vcpu);
1591	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
1592	kvm_rax_write(vcpu, eax);
1593	kvm_rbx_write(vcpu, ebx);
1594	kvm_rcx_write(vcpu, ecx);
1595	kvm_rdx_write(vcpu, edx);
1596	return kvm_skip_emulated_instruction(vcpu);
1597}
1598EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);