1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
   4 * Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
   5 *
   6 * KVM Xen emulation
   7 */
   8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10#include "x86.h"
  11#include "xen.h"
  12#include "hyperv.h"
  13#include "irq.h"
  14
  15#include <linux/eventfd.h>
  16#include <linux/kvm_host.h>
  17#include <linux/sched/stat.h>
  18
  19#include <trace/events/kvm.h>
  20#include <xen/interface/xen.h>
  21#include <xen/interface/vcpu.h>
  22#include <xen/interface/version.h>
  23#include <xen/interface/event_channel.h>
  24#include <xen/interface/sched.h>
  25
  26#include <asm/xen/cpuid.h>
  27#include <asm/pvclock.h>
  28
  29#include "cpuid.h"
  30#include "trace.h"
  31
  32static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm);
  33static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data);
  34static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r);
  35
  36DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ);
  37
  38static int kvm_xen_shared_info_init(struct kvm *kvm)
  39{
  40	struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
  41	struct pvclock_wall_clock *wc;
  42	u32 *wc_sec_hi;
  43	u32 wc_version;
  44	u64 wall_nsec;
  45	int ret = 0;
  46	int idx = srcu_read_lock(&kvm->srcu);
  47
  48	read_lock_irq(&gpc->lock);
  49	while (!kvm_gpc_check(gpc, PAGE_SIZE)) {
  50		read_unlock_irq(&gpc->lock);
  51
  52		ret = kvm_gpc_refresh(gpc, PAGE_SIZE);
  53		if (ret)
  54			goto out;
  55
  56		read_lock_irq(&gpc->lock);
  57	}
  58
  59	/*
  60	 * This code mirrors kvm_write_wall_clock() except that it writes
  61	 * directly through the pfn cache and doesn't mark the page dirty.
  62	 */
  63	wall_nsec = kvm_get_wall_clock_epoch(kvm);
  64
  65	/* Paranoia checks on the 32-bit struct layout */
  66	BUILD_BUG_ON(offsetof(struct compat_shared_info, wc) != 0x900);
  67	BUILD_BUG_ON(offsetof(struct compat_shared_info, arch.wc_sec_hi) != 0x924);
  68	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
  69
  70#ifdef CONFIG_X86_64
  71	/* Paranoia checks on the 64-bit struct layout */
  72	BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00);
  73	BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c);
  74
  75	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
  76		struct shared_info *shinfo = gpc->khva;
  77
  78		wc_sec_hi = &shinfo->wc_sec_hi;
  79		wc = &shinfo->wc;
  80	} else
  81#endif
  82	{
  83		struct compat_shared_info *shinfo = gpc->khva;
  84
  85		wc_sec_hi = &shinfo->arch.wc_sec_hi;
  86		wc = &shinfo->wc;
  87	}
  88
  89	/* Increment and ensure an odd value */
  90	wc_version = wc->version = (wc->version + 1) | 1;
  91	smp_wmb();
  92
  93	wc->nsec = do_div(wall_nsec, NSEC_PER_SEC);
  94	wc->sec = (u32)wall_nsec;
  95	*wc_sec_hi = wall_nsec >> 32;
  96	smp_wmb();
  97
  98	wc->version = wc_version + 1;
  99	read_unlock_irq(&gpc->lock);
 100
 101	kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
 102
 103out:
 104	srcu_read_unlock(&kvm->srcu, idx);
 105	return ret;
 106}
 107
 108void kvm_xen_inject_timer_irqs(struct kvm_vcpu *vcpu)
 109{
 110	if (atomic_read(&vcpu->arch.xen.timer_pending) > 0) {
 111		struct kvm_xen_evtchn e;
 112
 113		e.vcpu_id = vcpu->vcpu_id;
 114		e.vcpu_idx = vcpu->vcpu_idx;
 115		e.port = vcpu->arch.xen.timer_virq;
 116		e.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
 117
 118		kvm_xen_set_evtchn(&e, vcpu->kvm);
 119
 120		vcpu->arch.xen.timer_expires = 0;
 121		atomic_set(&vcpu->arch.xen.timer_pending, 0);
 122	}
 123}
 124
 125static enum hrtimer_restart xen_timer_callback(struct hrtimer *timer)
 126{
 127	struct kvm_vcpu *vcpu = container_of(timer, struct kvm_vcpu,
 128					     arch.xen.timer);
 129	struct kvm_xen_evtchn e;
 130	int rc;
 131
 132	if (atomic_read(&vcpu->arch.xen.timer_pending))
 133		return HRTIMER_NORESTART;
 134
 135	e.vcpu_id = vcpu->vcpu_id;
 136	e.vcpu_idx = vcpu->vcpu_idx;
 137	e.port = vcpu->arch.xen.timer_virq;
 138	e.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
 139
 140	rc = kvm_xen_set_evtchn_fast(&e, vcpu->kvm);
 141	if (rc != -EWOULDBLOCK) {
 142		vcpu->arch.xen.timer_expires = 0;
 143		return HRTIMER_NORESTART;
 144	}
 145
 146	atomic_inc(&vcpu->arch.xen.timer_pending);
 147	kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
 148	kvm_vcpu_kick(vcpu);
 149
 150	return HRTIMER_NORESTART;
 151}
 152
 153static void kvm_xen_start_timer(struct kvm_vcpu *vcpu, u64 guest_abs,
 154				bool linux_wa)
 155{
 156	int64_t kernel_now, delta;
 157	uint64_t guest_now;
 158
 159	/*
 160	 * The guest provides the requested timeout in absolute nanoseconds
 161	 * of the KVM clock — as *it* sees it, based on the scaled TSC and
 162	 * the pvclock information provided by KVM.
 163	 *
 164	 * The kernel doesn't support hrtimers based on CLOCK_MONOTONIC_RAW
 165	 * so use CLOCK_MONOTONIC. In the timescales covered by timers, the
 166	 * difference won't matter much as there is no cumulative effect.
 167	 *
 168	 * Calculate the time for some arbitrary point in time around "now"
 169	 * in terms of both kvmclock and CLOCK_MONOTONIC. Calculate the
 170	 * delta between the kvmclock "now" value and the guest's requested
 171	 * timeout, apply the "Linux workaround" described below, and add
 172	 * the resulting delta to the CLOCK_MONOTONIC "now" value, to get
 173	 * the absolute CLOCK_MONOTONIC time at which the timer should
 174	 * fire.
 175	 */
 176	if (vcpu->arch.hv_clock.version && vcpu->kvm->arch.use_master_clock &&
 177	    static_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 178		uint64_t host_tsc, guest_tsc;
 179
 180		if (!IS_ENABLED(CONFIG_64BIT) ||
 181		    !kvm_get_monotonic_and_clockread(&kernel_now, &host_tsc)) {
 182			/*
 183			 * Don't fall back to get_kvmclock_ns() because it's
 184			 * broken; it has a systemic error in its results
 185			 * because it scales directly from host TSC to
 186			 * nanoseconds, and doesn't scale first to guest TSC
 187			 * and *then* to nanoseconds as the guest does.
 188			 *
 189			 * There is a small error introduced here because time
 190			 * continues to elapse between the ktime_get() and the
 191			 * subsequent rdtsc(). But not the systemic drift due
 192			 * to get_kvmclock_ns().
 193			 */
 194			kernel_now = ktime_get(); /* This is CLOCK_MONOTONIC */
 195			host_tsc = rdtsc();
 196		}
 197
 198		/* Calculate the guest kvmclock as the guest would do it. */
 199		guest_tsc = kvm_read_l1_tsc(vcpu, host_tsc);
 200		guest_now = __pvclock_read_cycles(&vcpu->arch.hv_clock,
 201						  guest_tsc);
 202	} else {
 203		/*
 204		 * Without CONSTANT_TSC, get_kvmclock_ns() is the only option.
 205		 *
 206		 * Also if the guest PV clock hasn't been set up yet, as is
 207		 * likely to be the case during migration when the vCPU has
 208		 * not been run yet. It would be possible to calculate the
 209		 * scaling factors properly in that case but there's not much
 210		 * point in doing so. The get_kvmclock_ns() drift accumulates
 211		 * over time, so it's OK to use it at startup. Besides, on
 212		 * migration there's going to be a little bit of skew in the
 213		 * precise moment at which timers fire anyway. Often they'll
 214		 * be in the "past" by the time the VM is running again after
 215		 * migration.
 216		 */
 217		guest_now = get_kvmclock_ns(vcpu->kvm);
 218		kernel_now = ktime_get();
 219	}
 220
 221	delta = guest_abs - guest_now;
 222
 223	/*
 224	 * Xen has a 'Linux workaround' in do_set_timer_op() which checks for
 225	 * negative absolute timeout values (caused by integer overflow), and
 226	 * for values about 13 days in the future (2^50ns) which would be
 227	 * caused by jiffies overflow. For those cases, Xen sets the timeout
 228	 * 100ms in the future (not *too* soon, since if a guest really did
 229	 * set a long timeout on purpose we don't want to keep churning CPU
 230	 * time by waking it up).  Emulate Xen's workaround when starting the
 231	 * timer in response to __HYPERVISOR_set_timer_op.
 232	 */
 233	if (linux_wa &&
 234	    unlikely((int64_t)guest_abs < 0 ||
 235		     (delta > 0 && (uint32_t) (delta >> 50) != 0))) {
 236		delta = 100 * NSEC_PER_MSEC;
 237		guest_abs = guest_now + delta;
 238	}
 239
 240	/*
 241	 * Avoid races with the old timer firing. Checking timer_expires
 242	 * to avoid calling hrtimer_cancel() will only have false positives
 243	 * so is fine.
 244	 */
 245	if (vcpu->arch.xen.timer_expires)
 246		hrtimer_cancel(&vcpu->arch.xen.timer);
 247
 248	atomic_set(&vcpu->arch.xen.timer_pending, 0);
 249	vcpu->arch.xen.timer_expires = guest_abs;
 250
 251	if (delta <= 0)
 252		xen_timer_callback(&vcpu->arch.xen.timer);
 253	else
 254		hrtimer_start(&vcpu->arch.xen.timer,
 255			      ktime_add_ns(kernel_now, delta),
 256			      HRTIMER_MODE_ABS_HARD);
 257}
 258
 259static void kvm_xen_stop_timer(struct kvm_vcpu *vcpu)
 260{
 261	hrtimer_cancel(&vcpu->arch.xen.timer);
 262	vcpu->arch.xen.timer_expires = 0;
 263	atomic_set(&vcpu->arch.xen.timer_pending, 0);
 264}
 265
 
 
 
 
 
 
 
 266static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic)
 267{
 268	struct kvm_vcpu_xen *vx = &v->arch.xen;
 269	struct gfn_to_pfn_cache *gpc1 = &vx->runstate_cache;
 270	struct gfn_to_pfn_cache *gpc2 = &vx->runstate2_cache;
 271	size_t user_len, user_len1, user_len2;
 272	struct vcpu_runstate_info rs;
 273	unsigned long flags;
 274	size_t times_ofs;
 275	uint8_t *update_bit = NULL;
 276	uint64_t entry_time;
 277	uint64_t *rs_times;
 278	int *rs_state;
 279
 280	/*
 281	 * The only difference between 32-bit and 64-bit versions of the
 282	 * runstate struct is the alignment of uint64_t in 32-bit, which
 283	 * means that the 64-bit version has an additional 4 bytes of
 284	 * padding after the first field 'state'. Let's be really really
 285	 * paranoid about that, and matching it with our internal data
 286	 * structures that we memcpy into it...
 287	 */
 288	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != 0);
 289	BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state) != 0);
 290	BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c);
 291#ifdef CONFIG_X86_64
 292	/*
 293	 * The 64-bit structure has 4 bytes of padding before 'state_entry_time'
 294	 * so each subsequent field is shifted by 4, and it's 4 bytes longer.
 295	 */
 296	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
 297		     offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4);
 298	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) !=
 299		     offsetof(struct compat_vcpu_runstate_info, time) + 4);
 300	BUILD_BUG_ON(sizeof(struct vcpu_runstate_info) != 0x2c + 4);
 301#endif
 302	/*
 303	 * The state field is in the same place at the start of both structs,
 304	 * and is the same size (int) as vx->current_runstate.
 305	 */
 306	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) !=
 307		     offsetof(struct compat_vcpu_runstate_info, state));
 308	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state) !=
 309		     sizeof(vx->current_runstate));
 310	BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) !=
 311		     sizeof(vx->current_runstate));
 312
 313	/*
 314	 * The state_entry_time field is 64 bits in both versions, and the
 315	 * XEN_RUNSTATE_UPDATE flag is in the top bit, which given that x86
 316	 * is little-endian means that it's in the last *byte* of the word.
 317	 * That detail is important later.
 318	 */
 319	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state_entry_time) !=
 320		     sizeof(uint64_t));
 321	BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) !=
 322		     sizeof(uint64_t));
 323	BUILD_BUG_ON((XEN_RUNSTATE_UPDATE >> 56) != 0x80);
 324
 325	/*
 326	 * The time array is four 64-bit quantities in both versions, matching
 327	 * the vx->runstate_times and immediately following state_entry_time.
 328	 */
 329	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
 330		     offsetof(struct vcpu_runstate_info, time) - sizeof(uint64_t));
 331	BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state_entry_time) !=
 332		     offsetof(struct compat_vcpu_runstate_info, time) - sizeof(uint64_t));
 333	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
 334		     sizeof_field(struct compat_vcpu_runstate_info, time));
 335	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
 336		     sizeof(vx->runstate_times));
 337
 338	if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
 339		user_len = sizeof(struct vcpu_runstate_info);
 340		times_ofs = offsetof(struct vcpu_runstate_info,
 341				     state_entry_time);
 342	} else {
 343		user_len = sizeof(struct compat_vcpu_runstate_info);
 344		times_ofs = offsetof(struct compat_vcpu_runstate_info,
 345				     state_entry_time);
 346	}
 347
 348	/*
 349	 * There are basically no alignment constraints. The guest can set it
 350	 * up so it crosses from one page to the next, and at arbitrary byte
 351	 * alignment (and the 32-bit ABI doesn't align the 64-bit integers
 352	 * anyway, even if the overall struct had been 64-bit aligned).
 353	 */
 354	if ((gpc1->gpa & ~PAGE_MASK) + user_len >= PAGE_SIZE) {
 355		user_len1 = PAGE_SIZE - (gpc1->gpa & ~PAGE_MASK);
 356		user_len2 = user_len - user_len1;
 357	} else {
 358		user_len1 = user_len;
 359		user_len2 = 0;
 360	}
 361	BUG_ON(user_len1 + user_len2 != user_len);
 362
 363 retry:
 364	/*
 365	 * Attempt to obtain the GPC lock on *both* (if there are two)
 366	 * gfn_to_pfn caches that cover the region.
 367	 */
 368	if (atomic) {
 369		local_irq_save(flags);
 370		if (!read_trylock(&gpc1->lock)) {
 371			local_irq_restore(flags);
 372			return;
 373		}
 374	} else {
 375		read_lock_irqsave(&gpc1->lock, flags);
 376	}
 377	while (!kvm_gpc_check(gpc1, user_len1)) {
 378		read_unlock_irqrestore(&gpc1->lock, flags);
 379
 380		/* When invoked from kvm_sched_out() we cannot sleep */
 381		if (atomic)
 382			return;
 383
 384		if (kvm_gpc_refresh(gpc1, user_len1))
 385			return;
 386
 387		read_lock_irqsave(&gpc1->lock, flags);
 388	}
 389
 390	if (likely(!user_len2)) {
 391		/*
 392		 * Set up three pointers directly to the runstate_info
 393		 * struct in the guest (via the GPC).
 394		 *
 395		 *  • @rs_state   → state field
 396		 *  • @rs_times   → state_entry_time field.
 397		 *  • @update_bit → last byte of state_entry_time, which
 398		 *                  contains the XEN_RUNSTATE_UPDATE bit.
 399		 */
 400		rs_state = gpc1->khva;
 401		rs_times = gpc1->khva + times_ofs;
 402		if (v->kvm->arch.xen.runstate_update_flag)
 403			update_bit = ((void *)(&rs_times[1])) - 1;
 404	} else {
 405		/*
 406		 * The guest's runstate_info is split across two pages and we
 407		 * need to hold and validate both GPCs simultaneously. We can
 408		 * declare a lock ordering GPC1 > GPC2 because nothing else
 409		 * takes them more than one at a time. Set a subclass on the
 410		 * gpc1 lock to make lockdep shut up about it.
 411		 */
 412		lock_set_subclass(&gpc1->lock.dep_map, 1, _THIS_IP_);
 413		if (atomic) {
 414			if (!read_trylock(&gpc2->lock)) {
 415				read_unlock_irqrestore(&gpc1->lock, flags);
 416				return;
 417			}
 418		} else {
 419			read_lock(&gpc2->lock);
 420		}
 421
 422		if (!kvm_gpc_check(gpc2, user_len2)) {
 423			read_unlock(&gpc2->lock);
 424			read_unlock_irqrestore(&gpc1->lock, flags);
 425
 426			/* When invoked from kvm_sched_out() we cannot sleep */
 427			if (atomic)
 428				return;
 429
 430			/*
 431			 * Use kvm_gpc_activate() here because if the runstate
 432			 * area was configured in 32-bit mode and only extends
 433			 * to the second page now because the guest changed to
 434			 * 64-bit mode, the second GPC won't have been set up.
 435			 */
 436			if (kvm_gpc_activate(gpc2, gpc1->gpa + user_len1,
 437					     user_len2))
 438				return;
 439
 440			/*
 441			 * We dropped the lock on GPC1 so we have to go all the
 442			 * way back and revalidate that too.
 443			 */
 444			goto retry;
 445		}
 446
 447		/*
 448		 * In this case, the runstate_info struct will be assembled on
 449		 * the kernel stack (compat or not as appropriate) and will
 450		 * be copied to GPC1/GPC2 with a dual memcpy. Set up the three
 451		 * rs pointers accordingly.
 452		 */
 453		rs_times = &rs.state_entry_time;
 454
 455		/*
 456		 * The rs_state pointer points to the start of what we'll
 457		 * copy to the guest, which in the case of a compat guest
 458		 * is the 32-bit field that the compiler thinks is padding.
 459		 */
 460		rs_state = ((void *)rs_times) - times_ofs;
 461
 462		/*
 463		 * The update_bit is still directly in the guest memory,
 464		 * via one GPC or the other.
 465		 */
 466		if (v->kvm->arch.xen.runstate_update_flag) {
 467			if (user_len1 >= times_ofs + sizeof(uint64_t))
 468				update_bit = gpc1->khva + times_ofs +
 469					sizeof(uint64_t) - 1;
 470			else
 471				update_bit = gpc2->khva + times_ofs +
 472					sizeof(uint64_t) - 1 - user_len1;
 473		}
 474
 475#ifdef CONFIG_X86_64
 476		/*
 477		 * Don't leak kernel memory through the padding in the 64-bit
 478		 * version of the struct.
 479		 */
 480		memset(&rs, 0, offsetof(struct vcpu_runstate_info, state_entry_time));
 481#endif
 482	}
 483
 484	/*
 485	 * First, set the XEN_RUNSTATE_UPDATE bit in the top bit of the
 486	 * state_entry_time field, directly in the guest. We need to set
 487	 * that (and write-barrier) before writing to the rest of the
 488	 * structure, and clear it last. Just as Xen does, we address the
 489	 * single *byte* in which it resides because it might be in a
 490	 * different cache line to the rest of the 64-bit word, due to
 491	 * the (lack of) alignment constraints.
 492	 */
 493	entry_time = vx->runstate_entry_time;
 494	if (update_bit) {
 495		entry_time |= XEN_RUNSTATE_UPDATE;
 496		*update_bit = (vx->runstate_entry_time | XEN_RUNSTATE_UPDATE) >> 56;
 497		smp_wmb();
 498	}
 499
 500	/*
 501	 * Now assemble the actual structure, either on our kernel stack
 502	 * or directly in the guest according to how the rs_state and
 503	 * rs_times pointers were set up above.
 504	 */
 505	*rs_state = vx->current_runstate;
 506	rs_times[0] = entry_time;
 507	memcpy(rs_times + 1, vx->runstate_times, sizeof(vx->runstate_times));
 508
 509	/* For the split case, we have to then copy it to the guest. */
 510	if (user_len2) {
 511		memcpy(gpc1->khva, rs_state, user_len1);
 512		memcpy(gpc2->khva, ((void *)rs_state) + user_len1, user_len2);
 513	}
 514	smp_wmb();
 515
 516	/* Finally, clear the XEN_RUNSTATE_UPDATE bit. */
 517	if (update_bit) {
 518		entry_time &= ~XEN_RUNSTATE_UPDATE;
 519		*update_bit = entry_time >> 56;
 520		smp_wmb();
 521	}
 522
 523	if (user_len2) {
 524		kvm_gpc_mark_dirty_in_slot(gpc2);
 525		read_unlock(&gpc2->lock);
 526	}
 527
 528	kvm_gpc_mark_dirty_in_slot(gpc1);
 529	read_unlock_irqrestore(&gpc1->lock, flags);
 530}
 531
 532void kvm_xen_update_runstate(struct kvm_vcpu *v, int state)
 533{
 534	struct kvm_vcpu_xen *vx = &v->arch.xen;
 535	u64 now = get_kvmclock_ns(v->kvm);
 536	u64 delta_ns = now - vx->runstate_entry_time;
 537	u64 run_delay = current->sched_info.run_delay;
 538
 539	if (unlikely(!vx->runstate_entry_time))
 540		vx->current_runstate = RUNSTATE_offline;
 541
 542	/*
 543	 * Time waiting for the scheduler isn't "stolen" if the
 544	 * vCPU wasn't running anyway.
 545	 */
 546	if (vx->current_runstate == RUNSTATE_running) {
 547		u64 steal_ns = run_delay - vx->last_steal;
 548
 549		delta_ns -= steal_ns;
 550
 551		vx->runstate_times[RUNSTATE_runnable] += steal_ns;
 552	}
 553	vx->last_steal = run_delay;
 554
 555	vx->runstate_times[vx->current_runstate] += delta_ns;
 556	vx->current_runstate = state;
 557	vx->runstate_entry_time = now;
 558
 559	if (vx->runstate_cache.active)
 560		kvm_xen_update_runstate_guest(v, state == RUNSTATE_runnable);
 561}
 562
 563void kvm_xen_inject_vcpu_vector(struct kvm_vcpu *v)
 564{
 565	struct kvm_lapic_irq irq = { };
 566
 567	irq.dest_id = v->vcpu_id;
 568	irq.vector = v->arch.xen.upcall_vector;
 569	irq.dest_mode = APIC_DEST_PHYSICAL;
 570	irq.shorthand = APIC_DEST_NOSHORT;
 571	irq.delivery_mode = APIC_DM_FIXED;
 572	irq.level = 1;
 573
 574	kvm_irq_delivery_to_apic(v->kvm, NULL, &irq, NULL);
 575}
 576
 577/*
 578 * On event channel delivery, the vcpu_info may not have been accessible.
 579 * In that case, there are bits in vcpu->arch.xen.evtchn_pending_sel which
 580 * need to be marked into the vcpu_info (and evtchn_upcall_pending set).
 581 * Do so now that we can sleep in the context of the vCPU to bring the
 582 * page in, and refresh the pfn cache for it.
 583 */
 584void kvm_xen_inject_pending_events(struct kvm_vcpu *v)
 585{
 586	unsigned long evtchn_pending_sel = READ_ONCE(v->arch.xen.evtchn_pending_sel);
 587	struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache;
 588	unsigned long flags;
 589
 590	if (!evtchn_pending_sel)
 591		return;
 592
 593	/*
 594	 * Yes, this is an open-coded loop. But that's just what put_user()
 595	 * does anyway. Page it in and retry the instruction. We're just a
 596	 * little more honest about it.
 597	 */
 598	read_lock_irqsave(&gpc->lock, flags);
 599	while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
 600		read_unlock_irqrestore(&gpc->lock, flags);
 601
 602		if (kvm_gpc_refresh(gpc, sizeof(struct vcpu_info)))
 603			return;
 604
 605		read_lock_irqsave(&gpc->lock, flags);
 606	}
 607
 608	/* Now gpc->khva is a valid kernel address for the vcpu_info */
 609	if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
 610		struct vcpu_info *vi = gpc->khva;
 611
 612		asm volatile(LOCK_PREFIX "orq %0, %1\n"
 613			     "notq %0\n"
 614			     LOCK_PREFIX "andq %0, %2\n"
 615			     : "=r" (evtchn_pending_sel),
 616			       "+m" (vi->evtchn_pending_sel),
 617			       "+m" (v->arch.xen.evtchn_pending_sel)
 618			     : "0" (evtchn_pending_sel));
 619		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
 620	} else {
 621		u32 evtchn_pending_sel32 = evtchn_pending_sel;
 622		struct compat_vcpu_info *vi = gpc->khva;
 623
 624		asm volatile(LOCK_PREFIX "orl %0, %1\n"
 625			     "notl %0\n"
 626			     LOCK_PREFIX "andl %0, %2\n"
 627			     : "=r" (evtchn_pending_sel32),
 628			       "+m" (vi->evtchn_pending_sel),
 629			       "+m" (v->arch.xen.evtchn_pending_sel)
 630			     : "0" (evtchn_pending_sel32));
 631		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
 632	}
 633
 634	kvm_gpc_mark_dirty_in_slot(gpc);
 635	read_unlock_irqrestore(&gpc->lock, flags);
 636
 637	/* For the per-vCPU lapic vector, deliver it as MSI. */
 638	if (v->arch.xen.upcall_vector)
 639		kvm_xen_inject_vcpu_vector(v);
 640}
 641
 642int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
 643{
 644	struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache;
 645	unsigned long flags;
 646	u8 rc = 0;
 647
 648	/*
 649	 * If the global upcall vector (HVMIRQ_callback_vector) is set and
 650	 * the vCPU's evtchn_upcall_pending flag is set, the IRQ is pending.
 651	 */
 652
 653	/* No need for compat handling here */
 654	BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
 655		     offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
 656	BUILD_BUG_ON(sizeof(rc) !=
 657		     sizeof_field(struct vcpu_info, evtchn_upcall_pending));
 658	BUILD_BUG_ON(sizeof(rc) !=
 659		     sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending));
 660
 661	read_lock_irqsave(&gpc->lock, flags);
 662	while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
 663		read_unlock_irqrestore(&gpc->lock, flags);
 664
 665		/*
 666		 * This function gets called from kvm_vcpu_block() after setting the
 667		 * task to TASK_INTERRUPTIBLE, to see if it needs to wake immediately
 668		 * from a HLT. So we really mustn't sleep. If the page ended up absent
 669		 * at that point, just return 1 in order to trigger an immediate wake,
 670		 * and we'll end up getting called again from a context where we *can*
 671		 * fault in the page and wait for it.
 672		 */
 673		if (in_atomic() || !task_is_running(current))
 674			return 1;
 675
 676		if (kvm_gpc_refresh(gpc, sizeof(struct vcpu_info))) {
 677			/*
 678			 * If this failed, userspace has screwed up the
 679			 * vcpu_info mapping. No interrupts for you.
 680			 */
 681			return 0;
 682		}
 683		read_lock_irqsave(&gpc->lock, flags);
 684	}
 685
 686	rc = ((struct vcpu_info *)gpc->khva)->evtchn_upcall_pending;
 687	read_unlock_irqrestore(&gpc->lock, flags);
 688	return rc;
 689}
 690
 691int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
 692{
 693	int r = -ENOENT;
 694
 695
 696	switch (data->type) {
 697	case KVM_XEN_ATTR_TYPE_LONG_MODE:
 698		if (!IS_ENABLED(CONFIG_64BIT) && data->u.long_mode) {
 699			r = -EINVAL;
 700		} else {
 701			mutex_lock(&kvm->arch.xen.xen_lock);
 702			kvm->arch.xen.long_mode = !!data->u.long_mode;
 703
 704			/*
 705			 * Re-initialize shared_info to put the wallclock in the
 706			 * correct place. Whilst it's not necessary to do this
 707			 * unless the mode is actually changed, it does no harm
 708			 * to make the call anyway.
 709			 */
 710			r = kvm->arch.xen.shinfo_cache.active ?
 711				kvm_xen_shared_info_init(kvm) : 0;
 712			mutex_unlock(&kvm->arch.xen.xen_lock);
 713		}
 714		break;
 715
 716	case KVM_XEN_ATTR_TYPE_SHARED_INFO:
 717	case KVM_XEN_ATTR_TYPE_SHARED_INFO_HVA: {
 718		int idx;
 719
 720		mutex_lock(&kvm->arch.xen.xen_lock);
 721
 722		idx = srcu_read_lock(&kvm->srcu);
 723
 724		if (data->type == KVM_XEN_ATTR_TYPE_SHARED_INFO) {
 725			gfn_t gfn = data->u.shared_info.gfn;
 726
 727			if (gfn == KVM_XEN_INVALID_GFN) {
 728				kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
 729				r = 0;
 730			} else {
 731				r = kvm_gpc_activate(&kvm->arch.xen.shinfo_cache,
 732						     gfn_to_gpa(gfn), PAGE_SIZE);
 733			}
 734		} else {
 735			void __user * hva = u64_to_user_ptr(data->u.shared_info.hva);
 736
 737			if (!PAGE_ALIGNED(hva)) {
 738				r = -EINVAL;
 739			} else if (!hva) {
 740				kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
 741				r = 0;
 742			} else {
 743				r = kvm_gpc_activate_hva(&kvm->arch.xen.shinfo_cache,
 744							 (unsigned long)hva, PAGE_SIZE);
 745			}
 746		}
 747
 748		srcu_read_unlock(&kvm->srcu, idx);
 749
 750		if (!r && kvm->arch.xen.shinfo_cache.active)
 751			r = kvm_xen_shared_info_init(kvm);
 752
 753		mutex_unlock(&kvm->arch.xen.xen_lock);
 754		break;
 755	}
 756	case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
 757		if (data->u.vector && data->u.vector < 0x10)
 758			r = -EINVAL;
 759		else {
 760			mutex_lock(&kvm->arch.xen.xen_lock);
 761			kvm->arch.xen.upcall_vector = data->u.vector;
 762			mutex_unlock(&kvm->arch.xen.xen_lock);
 763			r = 0;
 764		}
 765		break;
 766
 767	case KVM_XEN_ATTR_TYPE_EVTCHN:
 768		r = kvm_xen_setattr_evtchn(kvm, data);
 769		break;
 770
 771	case KVM_XEN_ATTR_TYPE_XEN_VERSION:
 772		mutex_lock(&kvm->arch.xen.xen_lock);
 773		kvm->arch.xen.xen_version = data->u.xen_version;
 774		mutex_unlock(&kvm->arch.xen.xen_lock);
 775		r = 0;
 776		break;
 777
 778	case KVM_XEN_ATTR_TYPE_RUNSTATE_UPDATE_FLAG:
 779		if (!sched_info_on()) {
 780			r = -EOPNOTSUPP;
 781			break;
 782		}
 783		mutex_lock(&kvm->arch.xen.xen_lock);
 784		kvm->arch.xen.runstate_update_flag = !!data->u.runstate_update_flag;
 785		mutex_unlock(&kvm->arch.xen.xen_lock);
 786		r = 0;
 787		break;
 788
 789	default:
 790		break;
 791	}
 792
 793	return r;
 794}
 795
 796int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
 797{
 798	int r = -ENOENT;
 799
 800	mutex_lock(&kvm->arch.xen.xen_lock);
 801
 802	switch (data->type) {
 803	case KVM_XEN_ATTR_TYPE_LONG_MODE:
 804		data->u.long_mode = kvm->arch.xen.long_mode;
 805		r = 0;
 806		break;
 807
 808	case KVM_XEN_ATTR_TYPE_SHARED_INFO:
 809		if (kvm_gpc_is_gpa_active(&kvm->arch.xen.shinfo_cache))
 810			data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa);
 811		else
 812			data->u.shared_info.gfn = KVM_XEN_INVALID_GFN;
 813		r = 0;
 814		break;
 815
 816	case KVM_XEN_ATTR_TYPE_SHARED_INFO_HVA:
 817		if (kvm_gpc_is_hva_active(&kvm->arch.xen.shinfo_cache))
 818			data->u.shared_info.hva = kvm->arch.xen.shinfo_cache.uhva;
 819		else
 820			data->u.shared_info.hva = 0;
 821		r = 0;
 822		break;
 823
 824	case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
 825		data->u.vector = kvm->arch.xen.upcall_vector;
 826		r = 0;
 827		break;
 828
 829	case KVM_XEN_ATTR_TYPE_XEN_VERSION:
 830		data->u.xen_version = kvm->arch.xen.xen_version;
 831		r = 0;
 832		break;
 833
 834	case KVM_XEN_ATTR_TYPE_RUNSTATE_UPDATE_FLAG:
 835		if (!sched_info_on()) {
 836			r = -EOPNOTSUPP;
 837			break;
 838		}
 839		data->u.runstate_update_flag = kvm->arch.xen.runstate_update_flag;
 840		r = 0;
 841		break;
 842
 843	default:
 844		break;
 845	}
 846
 847	mutex_unlock(&kvm->arch.xen.xen_lock);
 848	return r;
 849}
 850
 851int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
 852{
 853	int idx, r = -ENOENT;
 854
 855	mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
 856	idx = srcu_read_lock(&vcpu->kvm->srcu);
 857
 858	switch (data->type) {
 859	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
 860	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO_HVA:
 861		/* No compat necessary here. */
 862		BUILD_BUG_ON(sizeof(struct vcpu_info) !=
 863			     sizeof(struct compat_vcpu_info));
 864		BUILD_BUG_ON(offsetof(struct vcpu_info, time) !=
 865			     offsetof(struct compat_vcpu_info, time));
 866
 867		if (data->type == KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO) {
 868			if (data->u.gpa == KVM_XEN_INVALID_GPA) {
 869				kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_info_cache);
 870				r = 0;
 871				break;
 872			}
 873
 874			r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache,
 875					     data->u.gpa, sizeof(struct vcpu_info));
 876		} else {
 877			if (data->u.hva == 0) {
 878				kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_info_cache);
 879				r = 0;
 880				break;
 881			}
 882
 883			r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache,
 884						 data->u.hva, sizeof(struct vcpu_info));
 885		}
 886
 887		if (!r)
 888			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 889
 890		break;
 891
 892	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
 893		if (data->u.gpa == KVM_XEN_INVALID_GPA) {
 894			kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_time_info_cache);
 895			r = 0;
 896			break;
 897		}
 898
 899		r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_time_info_cache,
 900				     data->u.gpa,
 901				     sizeof(struct pvclock_vcpu_time_info));
 902		if (!r)
 903			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 904		break;
 905
 906	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR: {
 907		size_t sz, sz1, sz2;
 908
 909		if (!sched_info_on()) {
 910			r = -EOPNOTSUPP;
 911			break;
 912		}
 913		if (data->u.gpa == KVM_XEN_INVALID_GPA) {
 914			r = 0;
 915		deactivate_out:
 916			kvm_gpc_deactivate(&vcpu->arch.xen.runstate_cache);
 917			kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
 918			break;
 919		}
 920
 921		/*
 922		 * If the guest switches to 64-bit mode after setting the runstate
 923		 * address, that's actually OK. kvm_xen_update_runstate_guest()
 924		 * will cope.
 925		 */
 926		if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode)
 927			sz = sizeof(struct vcpu_runstate_info);
 928		else
 929			sz = sizeof(struct compat_vcpu_runstate_info);
 930
 931		/* How much fits in the (first) page? */
 932		sz1 = PAGE_SIZE - (data->u.gpa & ~PAGE_MASK);
 933		r = kvm_gpc_activate(&vcpu->arch.xen.runstate_cache,
 934				     data->u.gpa, sz1);
 935		if (r)
 936			goto deactivate_out;
 937
 938		/* Either map the second page, or deactivate the second GPC */
 939		if (sz1 >= sz) {
 940			kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
 941		} else {
 942			sz2 = sz - sz1;
 943			BUG_ON((data->u.gpa + sz1) & ~PAGE_MASK);
 944			r = kvm_gpc_activate(&vcpu->arch.xen.runstate2_cache,
 945					     data->u.gpa + sz1, sz2);
 946			if (r)
 947				goto deactivate_out;
 948		}
 949
 950		kvm_xen_update_runstate_guest(vcpu, false);
 951		break;
 952	}
 953	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT:
 954		if (!sched_info_on()) {
 955			r = -EOPNOTSUPP;
 956			break;
 957		}
 958		if (data->u.runstate.state > RUNSTATE_offline) {
 959			r = -EINVAL;
 960			break;
 961		}
 962
 963		kvm_xen_update_runstate(vcpu, data->u.runstate.state);
 964		r = 0;
 965		break;
 966
 967	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA:
 968		if (!sched_info_on()) {
 969			r = -EOPNOTSUPP;
 970			break;
 971		}
 972		if (data->u.runstate.state > RUNSTATE_offline) {
 973			r = -EINVAL;
 974			break;
 975		}
 976		if (data->u.runstate.state_entry_time !=
 977		    (data->u.runstate.time_running +
 978		     data->u.runstate.time_runnable +
 979		     data->u.runstate.time_blocked +
 980		     data->u.runstate.time_offline)) {
 981			r = -EINVAL;
 982			break;
 983		}
 984		if (get_kvmclock_ns(vcpu->kvm) <
 985		    data->u.runstate.state_entry_time) {
 986			r = -EINVAL;
 987			break;
 988		}
 989
 990		vcpu->arch.xen.current_runstate = data->u.runstate.state;
 991		vcpu->arch.xen.runstate_entry_time =
 992			data->u.runstate.state_entry_time;
 993		vcpu->arch.xen.runstate_times[RUNSTATE_running] =
 994			data->u.runstate.time_running;
 995		vcpu->arch.xen.runstate_times[RUNSTATE_runnable] =
 996			data->u.runstate.time_runnable;
 997		vcpu->arch.xen.runstate_times[RUNSTATE_blocked] =
 998			data->u.runstate.time_blocked;
 999		vcpu->arch.xen.runstate_times[RUNSTATE_offline] =
1000			data->u.runstate.time_offline;
1001		vcpu->arch.xen.last_steal = current->sched_info.run_delay;
1002		r = 0;
1003		break;
1004
1005	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST:
1006		if (!sched_info_on()) {
1007			r = -EOPNOTSUPP;
1008			break;
1009		}
1010		if (data->u.runstate.state > RUNSTATE_offline &&
1011		    data->u.runstate.state != (u64)-1) {
1012			r = -EINVAL;
1013			break;
1014		}
1015		/* The adjustment must add up */
1016		if (data->u.runstate.state_entry_time !=
1017		    (data->u.runstate.time_running +
1018		     data->u.runstate.time_runnable +
1019		     data->u.runstate.time_blocked +
1020		     data->u.runstate.time_offline)) {
1021			r = -EINVAL;
1022			break;
1023		}
1024
1025		if (get_kvmclock_ns(vcpu->kvm) <
1026		    (vcpu->arch.xen.runstate_entry_time +
1027		     data->u.runstate.state_entry_time)) {
1028			r = -EINVAL;
1029			break;
1030		}
1031
1032		vcpu->arch.xen.runstate_entry_time +=
1033			data->u.runstate.state_entry_time;
1034		vcpu->arch.xen.runstate_times[RUNSTATE_running] +=
1035			data->u.runstate.time_running;
1036		vcpu->arch.xen.runstate_times[RUNSTATE_runnable] +=
1037			data->u.runstate.time_runnable;
1038		vcpu->arch.xen.runstate_times[RUNSTATE_blocked] +=
1039			data->u.runstate.time_blocked;
1040		vcpu->arch.xen.runstate_times[RUNSTATE_offline] +=
1041			data->u.runstate.time_offline;
1042
1043		if (data->u.runstate.state <= RUNSTATE_offline)
1044			kvm_xen_update_runstate(vcpu, data->u.runstate.state);
1045		else if (vcpu->arch.xen.runstate_cache.active)
1046			kvm_xen_update_runstate_guest(vcpu, false);
1047		r = 0;
1048		break;
1049
1050	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID:
1051		if (data->u.vcpu_id >= KVM_MAX_VCPUS)
1052			r = -EINVAL;
1053		else {
1054			vcpu->arch.xen.vcpu_id = data->u.vcpu_id;
1055			r = 0;
1056		}
1057		break;
1058
1059	case KVM_XEN_VCPU_ATTR_TYPE_TIMER:
1060		if (data->u.timer.port &&
1061		    data->u.timer.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) {
1062			r = -EINVAL;
1063			break;
1064		}
1065
 
 
 
1066		/* Stop the timer (if it's running) before changing the vector */
1067		kvm_xen_stop_timer(vcpu);
1068		vcpu->arch.xen.timer_virq = data->u.timer.port;
1069
1070		/* Start the timer if the new value has a valid vector+expiry. */
1071		if (data->u.timer.port && data->u.timer.expires_ns)
1072			kvm_xen_start_timer(vcpu, data->u.timer.expires_ns, false);
1073
1074		r = 0;
1075		break;
1076
1077	case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR:
1078		if (data->u.vector && data->u.vector < 0x10)
1079			r = -EINVAL;
1080		else {
1081			vcpu->arch.xen.upcall_vector = data->u.vector;
1082			r = 0;
1083		}
1084		break;
1085
1086	default:
1087		break;
1088	}
1089
1090	srcu_read_unlock(&vcpu->kvm->srcu, idx);
1091	mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
1092	return r;
1093}
1094
1095int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
1096{
1097	int r = -ENOENT;
1098
1099	mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
1100
1101	switch (data->type) {
1102	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
1103		if (kvm_gpc_is_gpa_active(&vcpu->arch.xen.vcpu_info_cache))
1104			data->u.gpa = vcpu->arch.xen.vcpu_info_cache.gpa;
1105		else
1106			data->u.gpa = KVM_XEN_INVALID_GPA;
1107		r = 0;
1108		break;
1109
1110	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO_HVA:
1111		if (kvm_gpc_is_hva_active(&vcpu->arch.xen.vcpu_info_cache))
1112			data->u.hva = vcpu->arch.xen.vcpu_info_cache.uhva;
1113		else
1114			data->u.hva = 0;
1115		r = 0;
1116		break;
1117
1118	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
1119		if (vcpu->arch.xen.vcpu_time_info_cache.active)
1120			data->u.gpa = vcpu->arch.xen.vcpu_time_info_cache.gpa;
1121		else
1122			data->u.gpa = KVM_XEN_INVALID_GPA;
1123		r = 0;
1124		break;
1125
1126	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR:
1127		if (!sched_info_on()) {
1128			r = -EOPNOTSUPP;
1129			break;
1130		}
1131		if (vcpu->arch.xen.runstate_cache.active) {
1132			data->u.gpa = vcpu->arch.xen.runstate_cache.gpa;
1133			r = 0;
1134		}
1135		break;
1136
1137	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT:
1138		if (!sched_info_on()) {
1139			r = -EOPNOTSUPP;
1140			break;
1141		}
1142		data->u.runstate.state = vcpu->arch.xen.current_runstate;
1143		r = 0;
1144		break;
1145
1146	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA:
1147		if (!sched_info_on()) {
1148			r = -EOPNOTSUPP;
1149			break;
1150		}
1151		data->u.runstate.state = vcpu->arch.xen.current_runstate;
1152		data->u.runstate.state_entry_time =
1153			vcpu->arch.xen.runstate_entry_time;
1154		data->u.runstate.time_running =
1155			vcpu->arch.xen.runstate_times[RUNSTATE_running];
1156		data->u.runstate.time_runnable =
1157			vcpu->arch.xen.runstate_times[RUNSTATE_runnable];
1158		data->u.runstate.time_blocked =
1159			vcpu->arch.xen.runstate_times[RUNSTATE_blocked];
1160		data->u.runstate.time_offline =
1161			vcpu->arch.xen.runstate_times[RUNSTATE_offline];
1162		r = 0;
1163		break;
1164
1165	case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST:
1166		r = -EINVAL;
1167		break;
1168
1169	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID:
1170		data->u.vcpu_id = vcpu->arch.xen.vcpu_id;
1171		r = 0;
1172		break;
1173
1174	case KVM_XEN_VCPU_ATTR_TYPE_TIMER:
1175		/*
1176		 * Ensure a consistent snapshot of state is captured, with a
1177		 * timer either being pending, or the event channel delivered
1178		 * to the corresponding bit in the shared_info. Not still
1179		 * lurking in the timer_pending flag for deferred delivery.
1180		 * Purely as an optimisation, if the timer_expires field is
1181		 * zero, that means the timer isn't active (or even in the
1182		 * timer_pending flag) and there is no need to cancel it.
1183		 */
1184		if (vcpu->arch.xen.timer_expires) {
1185			hrtimer_cancel(&vcpu->arch.xen.timer);
1186			kvm_xen_inject_timer_irqs(vcpu);
1187		}
1188
1189		data->u.timer.port = vcpu->arch.xen.timer_virq;
1190		data->u.timer.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
1191		data->u.timer.expires_ns = vcpu->arch.xen.timer_expires;
1192
1193		/*
1194		 * The hrtimer may trigger and raise the IRQ immediately,
1195		 * while the returned state causes it to be set up and
1196		 * raised again on the destination system after migration.
1197		 * That's fine, as the guest won't even have had a chance
1198		 * to run and handle the interrupt. Asserting an already
1199		 * pending event channel is idempotent.
1200		 */
1201		if (vcpu->arch.xen.timer_expires)
1202			hrtimer_start_expires(&vcpu->arch.xen.timer,
1203					      HRTIMER_MODE_ABS_HARD);
1204
1205		r = 0;
1206		break;
1207
1208	case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR:
1209		data->u.vector = vcpu->arch.xen.upcall_vector;
1210		r = 0;
1211		break;
1212
1213	default:
1214		break;
1215	}
1216
1217	mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
1218	return r;
1219}
1220
1221int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data)
1222{
1223	struct kvm *kvm = vcpu->kvm;
1224	u32 page_num = data & ~PAGE_MASK;
1225	u64 page_addr = data & PAGE_MASK;
1226	bool lm = is_long_mode(vcpu);
1227	int r = 0;
1228
1229	mutex_lock(&kvm->arch.xen.xen_lock);
1230	if (kvm->arch.xen.long_mode != lm) {
1231		kvm->arch.xen.long_mode = lm;
1232
1233		/*
1234		 * Re-initialize shared_info to put the wallclock in the
1235		 * correct place.
1236		 */
1237		if (kvm->arch.xen.shinfo_cache.active &&
1238		    kvm_xen_shared_info_init(kvm))
1239			r = 1;
1240	}
1241	mutex_unlock(&kvm->arch.xen.xen_lock);
1242
1243	if (r)
1244		return r;
1245
1246	/*
1247	 * If Xen hypercall intercept is enabled, fill the hypercall
1248	 * page with VMCALL/VMMCALL instructions since that's what
1249	 * we catch. Else the VMM has provided the hypercall pages
1250	 * with instructions of its own choosing, so use those.
1251	 */
1252	if (kvm_xen_hypercall_enabled(kvm)) {
1253		u8 instructions[32];
1254		int i;
1255
1256		if (page_num)
1257			return 1;
1258
1259		/* mov imm32, %eax */
1260		instructions[0] = 0xb8;
1261
1262		/* vmcall / vmmcall */
1263		kvm_x86_call(patch_hypercall)(vcpu, instructions + 5);
1264
1265		/* ret */
1266		instructions[8] = 0xc3;
1267
1268		/* int3 to pad */
1269		memset(instructions + 9, 0xcc, sizeof(instructions) - 9);
1270
1271		for (i = 0; i < PAGE_SIZE / sizeof(instructions); i++) {
1272			*(u32 *)&instructions[1] = i;
1273			if (kvm_vcpu_write_guest(vcpu,
1274						 page_addr + (i * sizeof(instructions)),
1275						 instructions, sizeof(instructions)))
1276				return 1;
1277		}
1278	} else {
1279		/*
1280		 * Note, truncation is a non-issue as 'lm' is guaranteed to be
1281		 * false for a 32-bit kernel, i.e. when hva_t is only 4 bytes.
1282		 */
1283		hva_t blob_addr = lm ? kvm->arch.xen_hvm_config.blob_addr_64
1284				     : kvm->arch.xen_hvm_config.blob_addr_32;
1285		u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
1286				  : kvm->arch.xen_hvm_config.blob_size_32;
1287		u8 *page;
1288		int ret;
1289
1290		if (page_num >= blob_size)
1291			return 1;
1292
1293		blob_addr += page_num * PAGE_SIZE;
1294
1295		page = memdup_user((u8 __user *)blob_addr, PAGE_SIZE);
1296		if (IS_ERR(page))
1297			return PTR_ERR(page);
1298
1299		ret = kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE);
1300		kfree(page);
1301		if (ret)
1302			return 1;
1303	}
1304	return 0;
1305}
1306
1307int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc)
1308{
1309	/* Only some feature flags need to be *enabled* by userspace */
1310	u32 permitted_flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
1311		KVM_XEN_HVM_CONFIG_EVTCHN_SEND |
1312		KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE;
1313	u32 old_flags;
1314
1315	if (xhc->flags & ~permitted_flags)
1316		return -EINVAL;
1317
1318	/*
1319	 * With hypercall interception the kernel generates its own
1320	 * hypercall page so it must not be provided.
1321	 */
1322	if ((xhc->flags & KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) &&
1323	    (xhc->blob_addr_32 || xhc->blob_addr_64 ||
1324	     xhc->blob_size_32 || xhc->blob_size_64))
1325		return -EINVAL;
1326
1327	mutex_lock(&kvm->arch.xen.xen_lock);
1328
1329	if (xhc->msr && !kvm->arch.xen_hvm_config.msr)
1330		static_branch_inc(&kvm_xen_enabled.key);
1331	else if (!xhc->msr && kvm->arch.xen_hvm_config.msr)
1332		static_branch_slow_dec_deferred(&kvm_xen_enabled);
1333
1334	old_flags = kvm->arch.xen_hvm_config.flags;
1335	memcpy(&kvm->arch.xen_hvm_config, xhc, sizeof(*xhc));
1336
1337	mutex_unlock(&kvm->arch.xen.xen_lock);
1338
1339	if ((old_flags ^ xhc->flags) & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE)
1340		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
1341
1342	return 0;
1343}
1344
1345static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
1346{
1347	kvm_rax_write(vcpu, result);
1348	return kvm_skip_emulated_instruction(vcpu);
1349}
1350
1351static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
1352{
1353	struct kvm_run *run = vcpu->run;
1354
1355	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.xen.hypercall_rip)))
1356		return 1;
1357
1358	return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result);
1359}
1360
1361static inline int max_evtchn_port(struct kvm *kvm)
1362{
1363	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode)
1364		return EVTCHN_2L_NR_CHANNELS;
1365	else
1366		return COMPAT_EVTCHN_2L_NR_CHANNELS;
1367}
1368
1369static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports,
1370			       evtchn_port_t *ports)
1371{
1372	struct kvm *kvm = vcpu->kvm;
1373	struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1374	unsigned long *pending_bits;
1375	unsigned long flags;
1376	bool ret = true;
1377	int idx, i;
1378
1379	idx = srcu_read_lock(&kvm->srcu);
1380	read_lock_irqsave(&gpc->lock, flags);
1381	if (!kvm_gpc_check(gpc, PAGE_SIZE))
1382		goto out_rcu;
1383
1384	ret = false;
1385	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1386		struct shared_info *shinfo = gpc->khva;
1387		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1388	} else {
1389		struct compat_shared_info *shinfo = gpc->khva;
1390		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1391	}
1392
1393	for (i = 0; i < nr_ports; i++) {
1394		if (test_bit(ports[i], pending_bits)) {
1395			ret = true;
1396			break;
1397		}
1398	}
1399
1400 out_rcu:
1401	read_unlock_irqrestore(&gpc->lock, flags);
1402	srcu_read_unlock(&kvm->srcu, idx);
1403
1404	return ret;
1405}
1406
1407static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
1408				 u64 param, u64 *r)
1409{
1410	struct sched_poll sched_poll;
1411	evtchn_port_t port, *ports;
1412	struct x86_exception e;
1413	int i;
1414
1415	if (!lapic_in_kernel(vcpu) ||
1416	    !(vcpu->kvm->arch.xen_hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND))
1417		return false;
1418
1419	if (IS_ENABLED(CONFIG_64BIT) && !longmode) {
1420		struct compat_sched_poll sp32;
1421
1422		/* Sanity check that the compat struct definition is correct */
1423		BUILD_BUG_ON(sizeof(sp32) != 16);
1424
1425		if (kvm_read_guest_virt(vcpu, param, &sp32, sizeof(sp32), &e)) {
1426			*r = -EFAULT;
1427			return true;
1428		}
1429
1430		/*
1431		 * This is a 32-bit pointer to an array of evtchn_port_t which
1432		 * are uint32_t, so once it's converted no further compat
1433		 * handling is needed.
1434		 */
1435		sched_poll.ports = (void *)(unsigned long)(sp32.ports);
1436		sched_poll.nr_ports = sp32.nr_ports;
1437		sched_poll.timeout = sp32.timeout;
1438	} else {
1439		if (kvm_read_guest_virt(vcpu, param, &sched_poll,
1440					sizeof(sched_poll), &e)) {
1441			*r = -EFAULT;
1442			return true;
1443		}
1444	}
1445
1446	if (unlikely(sched_poll.nr_ports > 1)) {
1447		/* Xen (unofficially) limits number of pollers to 128 */
1448		if (sched_poll.nr_ports > 128) {
1449			*r = -EINVAL;
1450			return true;
1451		}
1452
1453		ports = kmalloc_array(sched_poll.nr_ports,
1454				      sizeof(*ports), GFP_KERNEL);
1455		if (!ports) {
1456			*r = -ENOMEM;
1457			return true;
1458		}
1459	} else
1460		ports = &port;
1461
1462	if (kvm_read_guest_virt(vcpu, (gva_t)sched_poll.ports, ports,
1463				sched_poll.nr_ports * sizeof(*ports), &e)) {
1464		*r = -EFAULT;
1465		return true;
1466	}
1467
1468	for (i = 0; i < sched_poll.nr_ports; i++) {
1469		if (ports[i] >= max_evtchn_port(vcpu->kvm)) {
1470			*r = -EINVAL;
1471			goto out;
1472		}
1473	}
1474
1475	if (sched_poll.nr_ports == 1)
1476		vcpu->arch.xen.poll_evtchn = port;
1477	else
1478		vcpu->arch.xen.poll_evtchn = -1;
1479
1480	set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
1481
1482	if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) {
1483		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
1484
1485		if (sched_poll.timeout)
1486			mod_timer(&vcpu->arch.xen.poll_timer,
1487				  jiffies + nsecs_to_jiffies(sched_poll.timeout));
1488
1489		kvm_vcpu_halt(vcpu);
1490
1491		if (sched_poll.timeout)
1492			del_timer(&vcpu->arch.xen.poll_timer);
1493
1494		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
1495	}
1496
1497	vcpu->arch.xen.poll_evtchn = 0;
1498	*r = 0;
1499out:
1500	/* Really, this is only needed in case of timeout */
1501	clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
1502
1503	if (unlikely(sched_poll.nr_ports > 1))
1504		kfree(ports);
1505	return true;
1506}
1507
1508static void cancel_evtchn_poll(struct timer_list *t)
1509{
1510	struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.xen.poll_timer);
1511
1512	kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1513	kvm_vcpu_kick(vcpu);
1514}
1515
1516static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode,
1517				   int cmd, u64 param, u64 *r)
1518{
1519	switch (cmd) {
1520	case SCHEDOP_poll:
1521		if (kvm_xen_schedop_poll(vcpu, longmode, param, r))
1522			return true;
1523		fallthrough;
1524	case SCHEDOP_yield:
1525		kvm_vcpu_on_spin(vcpu, true);
1526		*r = 0;
1527		return true;
1528	default:
1529		break;
1530	}
1531
1532	return false;
1533}
1534
1535struct compat_vcpu_set_singleshot_timer {
1536    uint64_t timeout_abs_ns;
1537    uint32_t flags;
1538} __attribute__((packed));
1539
1540static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
1541				  int vcpu_id, u64 param, u64 *r)
1542{
1543	struct vcpu_set_singleshot_timer oneshot;
1544	struct x86_exception e;
1545
1546	if (!kvm_xen_timer_enabled(vcpu))
1547		return false;
1548
1549	switch (cmd) {
1550	case VCPUOP_set_singleshot_timer:
1551		if (vcpu->arch.xen.vcpu_id != vcpu_id) {
1552			*r = -EINVAL;
1553			return true;
1554		}
1555
1556		/*
1557		 * The only difference for 32-bit compat is the 4 bytes of
1558		 * padding after the interesting part of the structure. So
1559		 * for a faithful emulation of Xen we have to *try* to copy
1560		 * the padding and return -EFAULT if we can't. Otherwise we
1561		 * might as well just have copied the 12-byte 32-bit struct.
1562		 */
1563		BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) !=
1564			     offsetof(struct vcpu_set_singleshot_timer, timeout_abs_ns));
1565		BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) !=
1566			     sizeof_field(struct vcpu_set_singleshot_timer, timeout_abs_ns));
1567		BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, flags) !=
1568			     offsetof(struct vcpu_set_singleshot_timer, flags));
1569		BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) !=
1570			     sizeof_field(struct vcpu_set_singleshot_timer, flags));
1571
1572		if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) :
1573					sizeof(struct compat_vcpu_set_singleshot_timer), &e)) {
1574			*r = -EFAULT;
1575			return true;
1576		}
1577
1578		kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, false);
1579		*r = 0;
1580		return true;
1581
1582	case VCPUOP_stop_singleshot_timer:
1583		if (vcpu->arch.xen.vcpu_id != vcpu_id) {
1584			*r = -EINVAL;
1585			return true;
1586		}
1587		kvm_xen_stop_timer(vcpu);
1588		*r = 0;
1589		return true;
1590	}
1591
1592	return false;
1593}
1594
1595static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout,
1596				       u64 *r)
1597{
1598	if (!kvm_xen_timer_enabled(vcpu))
1599		return false;
1600
1601	if (timeout)
1602		kvm_xen_start_timer(vcpu, timeout, true);
1603	else
1604		kvm_xen_stop_timer(vcpu);
1605
1606	*r = 0;
1607	return true;
1608}
1609
1610int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
1611{
1612	bool longmode;
1613	u64 input, params[6], r = -ENOSYS;
1614	bool handled = false;
1615	u8 cpl;
1616
1617	input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX);
1618
1619	/* Hyper-V hypercalls get bit 31 set in EAX */
1620	if ((input & 0x80000000) &&
1621	    kvm_hv_hypercall_enabled(vcpu))
1622		return kvm_hv_hypercall(vcpu);
1623
1624	longmode = is_64_bit_hypercall(vcpu);
1625	if (!longmode) {
1626		params[0] = (u32)kvm_rbx_read(vcpu);
1627		params[1] = (u32)kvm_rcx_read(vcpu);
1628		params[2] = (u32)kvm_rdx_read(vcpu);
1629		params[3] = (u32)kvm_rsi_read(vcpu);
1630		params[4] = (u32)kvm_rdi_read(vcpu);
1631		params[5] = (u32)kvm_rbp_read(vcpu);
1632	}
1633#ifdef CONFIG_X86_64
1634	else {
1635		params[0] = (u64)kvm_rdi_read(vcpu);
1636		params[1] = (u64)kvm_rsi_read(vcpu);
1637		params[2] = (u64)kvm_rdx_read(vcpu);
1638		params[3] = (u64)kvm_r10_read(vcpu);
1639		params[4] = (u64)kvm_r8_read(vcpu);
1640		params[5] = (u64)kvm_r9_read(vcpu);
1641	}
1642#endif
1643	cpl = kvm_x86_call(get_cpl)(vcpu);
1644	trace_kvm_xen_hypercall(cpl, input, params[0], params[1], params[2],
1645				params[3], params[4], params[5]);
1646
1647	/*
1648	 * Only allow hypercall acceleration for CPL0. The rare hypercalls that
1649	 * are permitted in guest userspace can be handled by the VMM.
1650	 */
1651	if (unlikely(cpl > 0))
1652		goto handle_in_userspace;
1653
1654	switch (input) {
1655	case __HYPERVISOR_xen_version:
1656		if (params[0] == XENVER_version && vcpu->kvm->arch.xen.xen_version) {
1657			r = vcpu->kvm->arch.xen.xen_version;
1658			handled = true;
1659		}
1660		break;
1661	case __HYPERVISOR_event_channel_op:
1662		if (params[0] == EVTCHNOP_send)
1663			handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r);
1664		break;
1665	case __HYPERVISOR_sched_op:
1666		handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0],
1667						 params[1], &r);
1668		break;
1669	case __HYPERVISOR_vcpu_op:
1670		handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1],
1671						params[2], &r);
1672		break;
1673	case __HYPERVISOR_set_timer_op: {
1674		u64 timeout = params[0];
1675		/* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */
1676		if (!longmode)
1677			timeout |= params[1] << 32;
1678		handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r);
1679		break;
1680	}
1681	default:
1682		break;
1683	}
1684
1685	if (handled)
1686		return kvm_xen_hypercall_set_result(vcpu, r);
1687
1688handle_in_userspace:
1689	vcpu->run->exit_reason = KVM_EXIT_XEN;
1690	vcpu->run->xen.type = KVM_EXIT_XEN_HCALL;
1691	vcpu->run->xen.u.hcall.longmode = longmode;
1692	vcpu->run->xen.u.hcall.cpl = cpl;
1693	vcpu->run->xen.u.hcall.input = input;
1694	vcpu->run->xen.u.hcall.params[0] = params[0];
1695	vcpu->run->xen.u.hcall.params[1] = params[1];
1696	vcpu->run->xen.u.hcall.params[2] = params[2];
1697	vcpu->run->xen.u.hcall.params[3] = params[3];
1698	vcpu->run->xen.u.hcall.params[4] = params[4];
1699	vcpu->run->xen.u.hcall.params[5] = params[5];
1700	vcpu->arch.xen.hypercall_rip = kvm_get_linear_rip(vcpu);
1701	vcpu->arch.complete_userspace_io =
1702		kvm_xen_hypercall_complete_userspace;
1703
1704	return 0;
1705}
1706
1707static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port)
1708{
1709	int poll_evtchn = vcpu->arch.xen.poll_evtchn;
1710
1711	if ((poll_evtchn == port || poll_evtchn == -1) &&
1712	    test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) {
1713		kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1714		kvm_vcpu_kick(vcpu);
1715	}
1716}
1717
1718/*
1719 * The return value from this function is propagated to kvm_set_irq() API,
1720 * so it returns:
1721 *  < 0   Interrupt was ignored (masked or not delivered for other reasons)
1722 *  = 0   Interrupt was coalesced (previous irq is still pending)
1723 *  > 0   Number of CPUs interrupt was delivered to
1724 *
1725 * It is also called directly from kvm_arch_set_irq_inatomic(), where the
1726 * only check on its return value is a comparison with -EWOULDBLOCK'.
1727 */
1728int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
1729{
1730	struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1731	struct kvm_vcpu *vcpu;
1732	unsigned long *pending_bits, *mask_bits;
1733	unsigned long flags;
1734	int port_word_bit;
1735	bool kick_vcpu = false;
1736	int vcpu_idx, idx, rc;
1737
1738	vcpu_idx = READ_ONCE(xe->vcpu_idx);
1739	if (vcpu_idx >= 0)
1740		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1741	else {
1742		vcpu = kvm_get_vcpu_by_id(kvm, xe->vcpu_id);
1743		if (!vcpu)
1744			return -EINVAL;
1745		WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx);
1746	}
1747
1748	if (xe->port >= max_evtchn_port(kvm))
1749		return -EINVAL;
1750
1751	rc = -EWOULDBLOCK;
1752
1753	idx = srcu_read_lock(&kvm->srcu);
1754
1755	read_lock_irqsave(&gpc->lock, flags);
1756	if (!kvm_gpc_check(gpc, PAGE_SIZE))
1757		goto out_rcu;
1758
1759	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1760		struct shared_info *shinfo = gpc->khva;
1761		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1762		mask_bits = (unsigned long *)&shinfo->evtchn_mask;
1763		port_word_bit = xe->port / 64;
1764	} else {
1765		struct compat_shared_info *shinfo = gpc->khva;
1766		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1767		mask_bits = (unsigned long *)&shinfo->evtchn_mask;
1768		port_word_bit = xe->port / 32;
1769	}
1770
1771	/*
1772	 * If this port wasn't already set, and if it isn't masked, then
1773	 * we try to set the corresponding bit in the in-kernel shadow of
1774	 * evtchn_pending_sel for the target vCPU. And if *that* wasn't
1775	 * already set, then we kick the vCPU in question to write to the
1776	 * *real* evtchn_pending_sel in its own guest vcpu_info struct.
1777	 */
1778	if (test_and_set_bit(xe->port, pending_bits)) {
1779		rc = 0; /* It was already raised */
1780	} else if (test_bit(xe->port, mask_bits)) {
1781		rc = -ENOTCONN; /* Masked */
1782		kvm_xen_check_poller(vcpu, xe->port);
1783	} else {
1784		rc = 1; /* Delivered to the bitmap in shared_info. */
1785		/* Now switch to the vCPU's vcpu_info to set the index and pending_sel */
1786		read_unlock_irqrestore(&gpc->lock, flags);
1787		gpc = &vcpu->arch.xen.vcpu_info_cache;
1788
1789		read_lock_irqsave(&gpc->lock, flags);
1790		if (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
1791			/*
1792			 * Could not access the vcpu_info. Set the bit in-kernel
1793			 * and prod the vCPU to deliver it for itself.
1794			 */
1795			if (!test_and_set_bit(port_word_bit, &vcpu->arch.xen.evtchn_pending_sel))
1796				kick_vcpu = true;
1797			goto out_rcu;
1798		}
1799
1800		if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1801			struct vcpu_info *vcpu_info = gpc->khva;
1802			if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) {
1803				WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
1804				kick_vcpu = true;
1805			}
1806		} else {
1807			struct compat_vcpu_info *vcpu_info = gpc->khva;
1808			if (!test_and_set_bit(port_word_bit,
1809					      (unsigned long *)&vcpu_info->evtchn_pending_sel)) {
1810				WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
1811				kick_vcpu = true;
1812			}
1813		}
1814
1815		/* For the per-vCPU lapic vector, deliver it as MSI. */
1816		if (kick_vcpu && vcpu->arch.xen.upcall_vector) {
1817			kvm_xen_inject_vcpu_vector(vcpu);
1818			kick_vcpu = false;
1819		}
1820	}
1821
1822 out_rcu:
1823	read_unlock_irqrestore(&gpc->lock, flags);
1824	srcu_read_unlock(&kvm->srcu, idx);
1825
1826	if (kick_vcpu) {
1827		kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1828		kvm_vcpu_kick(vcpu);
1829	}
1830
1831	return rc;
1832}
1833
1834static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm)
1835{
1836	bool mm_borrowed = false;
1837	int rc;
1838
1839	rc = kvm_xen_set_evtchn_fast(xe, kvm);
1840	if (rc != -EWOULDBLOCK)
1841		return rc;
1842
1843	if (current->mm != kvm->mm) {
1844		/*
1845		 * If not on a thread which already belongs to this KVM,
1846		 * we'd better be in the irqfd workqueue.
1847		 */
1848		if (WARN_ON_ONCE(current->mm))
1849			return -EINVAL;
1850
1851		kthread_use_mm(kvm->mm);
1852		mm_borrowed = true;
1853	}
1854
1855	/*
1856	 * It is theoretically possible for the page to be unmapped
1857	 * and the MMU notifier to invalidate the shared_info before
1858	 * we even get to use it. In that case, this looks like an
1859	 * infinite loop. It was tempting to do it via the userspace
1860	 * HVA instead... but that just *hides* the fact that it's
1861	 * an infinite loop, because if a fault occurs and it waits
1862	 * for the page to come back, it can *still* immediately
1863	 * fault and have to wait again, repeatedly.
1864	 *
1865	 * Conversely, the page could also have been reinstated by
1866	 * another thread before we even obtain the mutex above, so
1867	 * check again *first* before remapping it.
1868	 */
1869	do {
1870		struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1871		int idx;
1872
1873		rc = kvm_xen_set_evtchn_fast(xe, kvm);
1874		if (rc != -EWOULDBLOCK)
1875			break;
1876
1877		idx = srcu_read_lock(&kvm->srcu);
1878		rc = kvm_gpc_refresh(gpc, PAGE_SIZE);
1879		srcu_read_unlock(&kvm->srcu, idx);
1880	} while(!rc);
1881
1882	if (mm_borrowed)
1883		kthread_unuse_mm(kvm->mm);
1884
1885	return rc;
1886}
1887
1888/* This is the version called from kvm_set_irq() as the .set function */
1889static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
1890			 int irq_source_id, int level, bool line_status)
1891{
1892	if (!level)
1893		return -EINVAL;
1894
1895	return kvm_xen_set_evtchn(&e->xen_evtchn, kvm);
1896}
1897
1898/*
1899 * Set up an event channel interrupt from the KVM IRQ routing table.
1900 * Used for e.g. PIRQ from passed through physical devices.
1901 */
1902int kvm_xen_setup_evtchn(struct kvm *kvm,
1903			 struct kvm_kernel_irq_routing_entry *e,
1904			 const struct kvm_irq_routing_entry *ue)
1905
1906{
1907	struct kvm_vcpu *vcpu;
1908
1909	if (ue->u.xen_evtchn.port >= max_evtchn_port(kvm))
1910		return -EINVAL;
1911
1912	/* We only support 2 level event channels for now */
1913	if (ue->u.xen_evtchn.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1914		return -EINVAL;
1915
1916	/*
1917	 * Xen gives us interesting mappings from vCPU index to APIC ID,
1918	 * which means kvm_get_vcpu_by_id() has to iterate over all vCPUs
1919	 * to find it. Do that once at setup time, instead of every time.
1920	 * But beware that on live update / live migration, the routing
1921	 * table might be reinstated before the vCPU threads have finished
1922	 * recreating their vCPUs.
1923	 */
1924	vcpu = kvm_get_vcpu_by_id(kvm, ue->u.xen_evtchn.vcpu);
1925	if (vcpu)
1926		e->xen_evtchn.vcpu_idx = vcpu->vcpu_idx;
1927	else
1928		e->xen_evtchn.vcpu_idx = -1;
1929
1930	e->xen_evtchn.port = ue->u.xen_evtchn.port;
1931	e->xen_evtchn.vcpu_id = ue->u.xen_evtchn.vcpu;
1932	e->xen_evtchn.priority = ue->u.xen_evtchn.priority;
1933	e->set = evtchn_set_fn;
1934
1935	return 0;
1936}
1937
1938/*
1939 * Explicit event sending from userspace with KVM_XEN_HVM_EVTCHN_SEND ioctl.
1940 */
1941int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *uxe)
1942{
1943	struct kvm_xen_evtchn e;
1944	int ret;
1945
1946	if (!uxe->port || uxe->port >= max_evtchn_port(kvm))
1947		return -EINVAL;
1948
1949	/* We only support 2 level event channels for now */
1950	if (uxe->priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1951		return -EINVAL;
1952
1953	e.port = uxe->port;
1954	e.vcpu_id = uxe->vcpu;
1955	e.vcpu_idx = -1;
1956	e.priority = uxe->priority;
1957
1958	ret = kvm_xen_set_evtchn(&e, kvm);
1959
1960	/*
1961	 * None of that 'return 1 if it actually got delivered' nonsense.
1962	 * We don't care if it was masked (-ENOTCONN) either.
1963	 */
1964	if (ret > 0 || ret == -ENOTCONN)
1965		ret = 0;
1966
1967	return ret;
1968}
1969
1970/*
1971 * Support for *outbound* event channel events via the EVTCHNOP_send hypercall.
1972 */
1973struct evtchnfd {
1974	u32 send_port;
1975	u32 type;
1976	union {
1977		struct kvm_xen_evtchn port;
1978		struct {
1979			u32 port; /* zero */
1980			struct eventfd_ctx *ctx;
1981		} eventfd;
1982	} deliver;
1983};
1984
1985/*
1986 * Update target vCPU or priority for a registered sending channel.
1987 */
1988static int kvm_xen_eventfd_update(struct kvm *kvm,
1989				  struct kvm_xen_hvm_attr *data)
1990{
1991	u32 port = data->u.evtchn.send_port;
1992	struct evtchnfd *evtchnfd;
1993	int ret;
1994
1995	/* Protect writes to evtchnfd as well as the idr lookup.  */
1996	mutex_lock(&kvm->arch.xen.xen_lock);
1997	evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port);
1998
1999	ret = -ENOENT;
2000	if (!evtchnfd)
2001		goto out_unlock;
2002
2003	/* For an UPDATE, nothing may change except the priority/vcpu */
2004	ret = -EINVAL;
2005	if (evtchnfd->type != data->u.evtchn.type)
2006		goto out_unlock;
2007
2008	/*
2009	 * Port cannot change, and if it's zero that was an eventfd
2010	 * which can't be changed either.
2011	 */
2012	if (!evtchnfd->deliver.port.port ||
2013	    evtchnfd->deliver.port.port != data->u.evtchn.deliver.port.port)
2014		goto out_unlock;
2015
2016	/* We only support 2 level event channels for now */
2017	if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
2018		goto out_unlock;
2019
2020	evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority;
2021	if (evtchnfd->deliver.port.vcpu_id != data->u.evtchn.deliver.port.vcpu) {
2022		evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu;
2023		evtchnfd->deliver.port.vcpu_idx = -1;
2024	}
2025	ret = 0;
2026out_unlock:
2027	mutex_unlock(&kvm->arch.xen.xen_lock);
2028	return ret;
2029}
2030
2031/*
2032 * Configure the target (eventfd or local port delivery) for sending on
2033 * a given event channel.
2034 */
2035static int kvm_xen_eventfd_assign(struct kvm *kvm,
2036				  struct kvm_xen_hvm_attr *data)
2037{
2038	u32 port = data->u.evtchn.send_port;
2039	struct eventfd_ctx *eventfd = NULL;
2040	struct evtchnfd *evtchnfd;
2041	int ret = -EINVAL;
2042
2043	evtchnfd = kzalloc(sizeof(struct evtchnfd), GFP_KERNEL);
2044	if (!evtchnfd)
2045		return -ENOMEM;
2046
2047	switch(data->u.evtchn.type) {
2048	case EVTCHNSTAT_ipi:
2049		/* IPI  must map back to the same port# */
2050		if (data->u.evtchn.deliver.port.port != data->u.evtchn.send_port)
2051			goto out_noeventfd; /* -EINVAL */
2052		break;
2053
2054	case EVTCHNSTAT_interdomain:
2055		if (data->u.evtchn.deliver.port.port) {
2056			if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm))
2057				goto out_noeventfd; /* -EINVAL */
2058		} else {
2059			eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd);
2060			if (IS_ERR(eventfd)) {
2061				ret = PTR_ERR(eventfd);
2062				goto out_noeventfd;
2063			}
2064		}
2065		break;
2066
2067	case EVTCHNSTAT_virq:
2068	case EVTCHNSTAT_closed:
2069	case EVTCHNSTAT_unbound:
2070	case EVTCHNSTAT_pirq:
2071	default: /* Unknown event channel type */
2072		goto out; /* -EINVAL */
2073	}
2074
2075	evtchnfd->send_port = data->u.evtchn.send_port;
2076	evtchnfd->type = data->u.evtchn.type;
2077	if (eventfd) {
2078		evtchnfd->deliver.eventfd.ctx = eventfd;
2079	} else {
2080		/* We only support 2 level event channels for now */
2081		if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
2082			goto out; /* -EINVAL; */
2083
2084		evtchnfd->deliver.port.port = data->u.evtchn.deliver.port.port;
2085		evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu;
2086		evtchnfd->deliver.port.vcpu_idx = -1;
2087		evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority;
2088	}
2089
2090	mutex_lock(&kvm->arch.xen.xen_lock);
2091	ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1,
2092			GFP_KERNEL);
2093	mutex_unlock(&kvm->arch.xen.xen_lock);
2094	if (ret >= 0)
2095		return 0;
2096
2097	if (ret == -ENOSPC)
2098		ret = -EEXIST;
2099out:
2100	if (eventfd)
2101		eventfd_ctx_put(eventfd);
2102out_noeventfd:
2103	kfree(evtchnfd);
2104	return ret;
2105}
2106
2107static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port)
2108{
2109	struct evtchnfd *evtchnfd;
2110
2111	mutex_lock(&kvm->arch.xen.xen_lock);
2112	evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port);
2113	mutex_unlock(&kvm->arch.xen.xen_lock);
2114
2115	if (!evtchnfd)
2116		return -ENOENT;
2117
2118	synchronize_srcu(&kvm->srcu);
2119	if (!evtchnfd->deliver.port.port)
2120		eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
2121	kfree(evtchnfd);
2122	return 0;
2123}
2124
2125static int kvm_xen_eventfd_reset(struct kvm *kvm)
2126{
2127	struct evtchnfd *evtchnfd, **all_evtchnfds;
2128	int i;
2129	int n = 0;
2130
2131	mutex_lock(&kvm->arch.xen.xen_lock);
2132
2133	/*
2134	 * Because synchronize_srcu() cannot be called inside the
2135	 * critical section, first collect all the evtchnfd objects
2136	 * in an array as they are removed from evtchn_ports.
2137	 */
2138	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i)
2139		n++;
2140
2141	all_evtchnfds = kmalloc_array(n, sizeof(struct evtchnfd *), GFP_KERNEL);
2142	if (!all_evtchnfds) {
2143		mutex_unlock(&kvm->arch.xen.xen_lock);
2144		return -ENOMEM;
2145	}
2146
2147	n = 0;
2148	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
2149		all_evtchnfds[n++] = evtchnfd;
2150		idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port);
2151	}
2152	mutex_unlock(&kvm->arch.xen.xen_lock);
2153
2154	synchronize_srcu(&kvm->srcu);
2155
2156	while (n--) {
2157		evtchnfd = all_evtchnfds[n];
2158		if (!evtchnfd->deliver.port.port)
2159			eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
2160		kfree(evtchnfd);
2161	}
2162	kfree(all_evtchnfds);
2163
2164	return 0;
2165}
2166
2167static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
2168{
2169	u32 port = data->u.evtchn.send_port;
2170
2171	if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET)
2172		return kvm_xen_eventfd_reset(kvm);
2173
2174	if (!port || port >= max_evtchn_port(kvm))
2175		return -EINVAL;
2176
2177	if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN)
2178		return kvm_xen_eventfd_deassign(kvm, port);
2179	if (data->u.evtchn.flags == KVM_XEN_EVTCHN_UPDATE)
2180		return kvm_xen_eventfd_update(kvm, data);
2181	if (data->u.evtchn.flags)
2182		return -EINVAL;
2183
2184	return kvm_xen_eventfd_assign(kvm, data);
2185}
2186
2187static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r)
2188{
2189	struct evtchnfd *evtchnfd;
2190	struct evtchn_send send;
2191	struct x86_exception e;
2192
2193	/* Sanity check: this structure is the same for 32-bit and 64-bit */
2194	BUILD_BUG_ON(sizeof(send) != 4);
2195	if (kvm_read_guest_virt(vcpu, param, &send, sizeof(send), &e)) {
2196		*r = -EFAULT;
2197		return true;
2198	}
2199
2200	/*
2201	 * evtchnfd is protected by kvm->srcu; the idr lookup instead
2202	 * is protected by RCU.
2203	 */
2204	rcu_read_lock();
2205	evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port);
2206	rcu_read_unlock();
2207	if (!evtchnfd)
2208		return false;
2209
2210	if (evtchnfd->deliver.port.port) {
2211		int ret = kvm_xen_set_evtchn(&evtchnfd->deliver.port, vcpu->kvm);
2212		if (ret < 0 && ret != -ENOTCONN)
2213			return false;
2214	} else {
2215		eventfd_signal(evtchnfd->deliver.eventfd.ctx);
2216	}
2217
2218	*r = 0;
2219	return true;
2220}
2221
2222void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu)
2223{
2224	vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx;
2225	vcpu->arch.xen.poll_evtchn = 0;
2226
2227	timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0);
2228	hrtimer_init(&vcpu->arch.xen.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
2229	vcpu->arch.xen.timer.function = xen_timer_callback;
2230
2231	kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm);
2232	kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm);
2233	kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache, vcpu->kvm);
2234	kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache, vcpu->kvm);
2235}
2236
2237void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu)
2238{
2239	if (kvm_xen_timer_enabled(vcpu))
2240		kvm_xen_stop_timer(vcpu);
2241
2242	kvm_gpc_deactivate(&vcpu->arch.xen.runstate_cache);
2243	kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
2244	kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_info_cache);
2245	kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_time_info_cache);
2246
2247	del_timer_sync(&vcpu->arch.xen.poll_timer);
2248}
2249
2250void kvm_xen_update_tsc_info(struct kvm_vcpu *vcpu)
2251{
2252	struct kvm_cpuid_entry2 *entry;
2253	u32 function;
2254
2255	if (!vcpu->arch.xen.cpuid.base)
2256		return;
2257
2258	function = vcpu->arch.xen.cpuid.base | XEN_CPUID_LEAF(3);
2259	if (function > vcpu->arch.xen.cpuid.limit)
2260		return;
2261
2262	entry = kvm_find_cpuid_entry_index(vcpu, function, 1);
2263	if (entry) {
2264		entry->ecx = vcpu->arch.hv_clock.tsc_to_system_mul;
2265		entry->edx = vcpu->arch.hv_clock.tsc_shift;
2266	}
2267
2268	entry = kvm_find_cpuid_entry_index(vcpu, function, 2);
2269	if (entry)
2270		entry->eax = vcpu->arch.hw_tsc_khz;
2271}
2272
2273void kvm_xen_init_vm(struct kvm *kvm)
2274{
2275	mutex_init(&kvm->arch.xen.xen_lock);
2276	idr_init(&kvm->arch.xen.evtchn_ports);
2277	kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm);
2278}
2279
2280void kvm_xen_destroy_vm(struct kvm *kvm)
2281{
2282	struct evtchnfd *evtchnfd;
2283	int i;
2284
2285	kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
2286
2287	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
2288		if (!evtchnfd->deliver.port.port)
2289			eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
2290		kfree(evtchnfd);
2291	}
2292	idr_destroy(&kvm->arch.xen.evtchn_ports);
2293
2294	if (kvm->arch.xen_hvm_config.msr)
2295		static_branch_slow_dec_deferred(&kvm_xen_enabled);
2296}