Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
4 * Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
5 *
6 * KVM Xen emulation
7 */
8
9#include "x86.h"
10#include "xen.h"
11#include "hyperv.h"
12#include "lapic.h"
13
14#include <linux/eventfd.h>
15#include <linux/kvm_host.h>
16#include <linux/sched/stat.h>
17
18#include <trace/events/kvm.h>
19#include <xen/interface/xen.h>
20#include <xen/interface/vcpu.h>
21#include <xen/interface/version.h>
22#include <xen/interface/event_channel.h>
23#include <xen/interface/sched.h>
24
25#include "trace.h"
26
27static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm);
28static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data);
29static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r);
30
31DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ);
32
33static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn)
34{
35 struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
36 struct pvclock_wall_clock *wc;
37 gpa_t gpa = gfn_to_gpa(gfn);
38 u32 *wc_sec_hi;
39 u32 wc_version;
40 u64 wall_nsec;
41 int ret = 0;
42 int idx = srcu_read_lock(&kvm->srcu);
43
44 if (gfn == KVM_XEN_INVALID_GFN) {
45 kvm_gpc_deactivate(gpc);
46 goto out;
47 }
48
49 do {
50 ret = kvm_gpc_activate(gpc, gpa, PAGE_SIZE);
51 if (ret)
52 goto out;
53
54 /*
55 * This code mirrors kvm_write_wall_clock() except that it writes
56 * directly through the pfn cache and doesn't mark the page dirty.
57 */
58 wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
59
60 /* It could be invalid again already, so we need to check */
61 read_lock_irq(&gpc->lock);
62
63 if (gpc->valid)
64 break;
65
66 read_unlock_irq(&gpc->lock);
67 } while (1);
68
69 /* Paranoia checks on the 32-bit struct layout */
70 BUILD_BUG_ON(offsetof(struct compat_shared_info, wc) != 0x900);
71 BUILD_BUG_ON(offsetof(struct compat_shared_info, arch.wc_sec_hi) != 0x924);
72 BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
73
74#ifdef CONFIG_X86_64
75 /* Paranoia checks on the 64-bit struct layout */
76 BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00);
77 BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c);
78
79 if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
80 struct shared_info *shinfo = gpc->khva;
81
82 wc_sec_hi = &shinfo->wc_sec_hi;
83 wc = &shinfo->wc;
84 } else
85#endif
86 {
87 struct compat_shared_info *shinfo = gpc->khva;
88
89 wc_sec_hi = &shinfo->arch.wc_sec_hi;
90 wc = &shinfo->wc;
91 }
92
93 /* Increment and ensure an odd value */
94 wc_version = wc->version = (wc->version + 1) | 1;
95 smp_wmb();
96
97 wc->nsec = do_div(wall_nsec, 1000000000);
98 wc->sec = (u32)wall_nsec;
99 *wc_sec_hi = wall_nsec >> 32;
100 smp_wmb();
101
102 wc->version = wc_version + 1;
103 read_unlock_irq(&gpc->lock);
104
105 kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
106
107out:
108 srcu_read_unlock(&kvm->srcu, idx);
109 return ret;
110}
111
112void kvm_xen_inject_timer_irqs(struct kvm_vcpu *vcpu)
113{
114 if (atomic_read(&vcpu->arch.xen.timer_pending) > 0) {
115 struct kvm_xen_evtchn e;
116
117 e.vcpu_id = vcpu->vcpu_id;
118 e.vcpu_idx = vcpu->vcpu_idx;
119 e.port = vcpu->arch.xen.timer_virq;
120 e.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
121
122 kvm_xen_set_evtchn(&e, vcpu->kvm);
123
124 vcpu->arch.xen.timer_expires = 0;
125 atomic_set(&vcpu->arch.xen.timer_pending, 0);
126 }
127}
128
129static enum hrtimer_restart xen_timer_callback(struct hrtimer *timer)
130{
131 struct kvm_vcpu *vcpu = container_of(timer, struct kvm_vcpu,
132 arch.xen.timer);
133 if (atomic_read(&vcpu->arch.xen.timer_pending))
134 return HRTIMER_NORESTART;
135
136 atomic_inc(&vcpu->arch.xen.timer_pending);
137 kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
138 kvm_vcpu_kick(vcpu);
139
140 return HRTIMER_NORESTART;
141}
142
143static void kvm_xen_start_timer(struct kvm_vcpu *vcpu, u64 guest_abs, s64 delta_ns)
144{
145 atomic_set(&vcpu->arch.xen.timer_pending, 0);
146 vcpu->arch.xen.timer_expires = guest_abs;
147
148 if (delta_ns <= 0) {
149 xen_timer_callback(&vcpu->arch.xen.timer);
150 } else {
151 ktime_t ktime_now = ktime_get();
152 hrtimer_start(&vcpu->arch.xen.timer,
153 ktime_add_ns(ktime_now, delta_ns),
154 HRTIMER_MODE_ABS_HARD);
155 }
156}
157
158static void kvm_xen_stop_timer(struct kvm_vcpu *vcpu)
159{
160 hrtimer_cancel(&vcpu->arch.xen.timer);
161 vcpu->arch.xen.timer_expires = 0;
162 atomic_set(&vcpu->arch.xen.timer_pending, 0);
163}
164
165static void kvm_xen_init_timer(struct kvm_vcpu *vcpu)
166{
167 hrtimer_init(&vcpu->arch.xen.timer, CLOCK_MONOTONIC,
168 HRTIMER_MODE_ABS_HARD);
169 vcpu->arch.xen.timer.function = xen_timer_callback;
170}
171
172static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic)
173{
174 struct kvm_vcpu_xen *vx = &v->arch.xen;
175 struct gfn_to_pfn_cache *gpc1 = &vx->runstate_cache;
176 struct gfn_to_pfn_cache *gpc2 = &vx->runstate2_cache;
177 size_t user_len, user_len1, user_len2;
178 struct vcpu_runstate_info rs;
179 unsigned long flags;
180 size_t times_ofs;
181 uint8_t *update_bit = NULL;
182 uint64_t entry_time;
183 uint64_t *rs_times;
184 int *rs_state;
185
186 /*
187 * The only difference between 32-bit and 64-bit versions of the
188 * runstate struct is the alignment of uint64_t in 32-bit, which
189 * means that the 64-bit version has an additional 4 bytes of
190 * padding after the first field 'state'. Let's be really really
191 * paranoid about that, and matching it with our internal data
192 * structures that we memcpy into it...
193 */
194 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != 0);
195 BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state) != 0);
196 BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c);
197#ifdef CONFIG_X86_64
198 /*
199 * The 64-bit structure has 4 bytes of padding before 'state_entry_time'
200 * so each subsequent field is shifted by 4, and it's 4 bytes longer.
201 */
202 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
203 offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4);
204 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) !=
205 offsetof(struct compat_vcpu_runstate_info, time) + 4);
206 BUILD_BUG_ON(sizeof(struct vcpu_runstate_info) != 0x2c + 4);
207#endif
208 /*
209 * The state field is in the same place at the start of both structs,
210 * and is the same size (int) as vx->current_runstate.
211 */
212 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) !=
213 offsetof(struct compat_vcpu_runstate_info, state));
214 BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state) !=
215 sizeof(vx->current_runstate));
216 BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) !=
217 sizeof(vx->current_runstate));
218
219 /*
220 * The state_entry_time field is 64 bits in both versions, and the
221 * XEN_RUNSTATE_UPDATE flag is in the top bit, which given that x86
222 * is little-endian means that it's in the last *byte* of the word.
223 * That detail is important later.
224 */
225 BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state_entry_time) !=
226 sizeof(uint64_t));
227 BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) !=
228 sizeof(uint64_t));
229 BUILD_BUG_ON((XEN_RUNSTATE_UPDATE >> 56) != 0x80);
230
231 /*
232 * The time array is four 64-bit quantities in both versions, matching
233 * the vx->runstate_times and immediately following state_entry_time.
234 */
235 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
236 offsetof(struct vcpu_runstate_info, time) - sizeof(uint64_t));
237 BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state_entry_time) !=
238 offsetof(struct compat_vcpu_runstate_info, time) - sizeof(uint64_t));
239 BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
240 sizeof_field(struct compat_vcpu_runstate_info, time));
241 BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
242 sizeof(vx->runstate_times));
243
244 if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
245 user_len = sizeof(struct vcpu_runstate_info);
246 times_ofs = offsetof(struct vcpu_runstate_info,
247 state_entry_time);
248 } else {
249 user_len = sizeof(struct compat_vcpu_runstate_info);
250 times_ofs = offsetof(struct compat_vcpu_runstate_info,
251 state_entry_time);
252 }
253
254 /*
255 * There are basically no alignment constraints. The guest can set it
256 * up so it crosses from one page to the next, and at arbitrary byte
257 * alignment (and the 32-bit ABI doesn't align the 64-bit integers
258 * anyway, even if the overall struct had been 64-bit aligned).
259 */
260 if ((gpc1->gpa & ~PAGE_MASK) + user_len >= PAGE_SIZE) {
261 user_len1 = PAGE_SIZE - (gpc1->gpa & ~PAGE_MASK);
262 user_len2 = user_len - user_len1;
263 } else {
264 user_len1 = user_len;
265 user_len2 = 0;
266 }
267 BUG_ON(user_len1 + user_len2 != user_len);
268
269 retry:
270 /*
271 * Attempt to obtain the GPC lock on *both* (if there are two)
272 * gfn_to_pfn caches that cover the region.
273 */
274 if (atomic) {
275 local_irq_save(flags);
276 if (!read_trylock(&gpc1->lock)) {
277 local_irq_restore(flags);
278 return;
279 }
280 } else {
281 read_lock_irqsave(&gpc1->lock, flags);
282 }
283 while (!kvm_gpc_check(gpc1, user_len1)) {
284 read_unlock_irqrestore(&gpc1->lock, flags);
285
286 /* When invoked from kvm_sched_out() we cannot sleep */
287 if (atomic)
288 return;
289
290 if (kvm_gpc_refresh(gpc1, user_len1))
291 return;
292
293 read_lock_irqsave(&gpc1->lock, flags);
294 }
295
296 if (likely(!user_len2)) {
297 /*
298 * Set up three pointers directly to the runstate_info
299 * struct in the guest (via the GPC).
300 *
301 * • @rs_state → state field
302 * • @rs_times → state_entry_time field.
303 * • @update_bit → last byte of state_entry_time, which
304 * contains the XEN_RUNSTATE_UPDATE bit.
305 */
306 rs_state = gpc1->khva;
307 rs_times = gpc1->khva + times_ofs;
308 if (v->kvm->arch.xen.runstate_update_flag)
309 update_bit = ((void *)(&rs_times[1])) - 1;
310 } else {
311 /*
312 * The guest's runstate_info is split across two pages and we
313 * need to hold and validate both GPCs simultaneously. We can
314 * declare a lock ordering GPC1 > GPC2 because nothing else
315 * takes them more than one at a time. Set a subclass on the
316 * gpc1 lock to make lockdep shut up about it.
317 */
318 lock_set_subclass(&gpc1->lock.dep_map, 1, _THIS_IP_);
319 if (atomic) {
320 if (!read_trylock(&gpc2->lock)) {
321 read_unlock_irqrestore(&gpc1->lock, flags);
322 return;
323 }
324 } else {
325 read_lock(&gpc2->lock);
326 }
327
328 if (!kvm_gpc_check(gpc2, user_len2)) {
329 read_unlock(&gpc2->lock);
330 read_unlock_irqrestore(&gpc1->lock, flags);
331
332 /* When invoked from kvm_sched_out() we cannot sleep */
333 if (atomic)
334 return;
335
336 /*
337 * Use kvm_gpc_activate() here because if the runstate
338 * area was configured in 32-bit mode and only extends
339 * to the second page now because the guest changed to
340 * 64-bit mode, the second GPC won't have been set up.
341 */
342 if (kvm_gpc_activate(gpc2, gpc1->gpa + user_len1,
343 user_len2))
344 return;
345
346 /*
347 * We dropped the lock on GPC1 so we have to go all the
348 * way back and revalidate that too.
349 */
350 goto retry;
351 }
352
353 /*
354 * In this case, the runstate_info struct will be assembled on
355 * the kernel stack (compat or not as appropriate) and will
356 * be copied to GPC1/GPC2 with a dual memcpy. Set up the three
357 * rs pointers accordingly.
358 */
359 rs_times = &rs.state_entry_time;
360
361 /*
362 * The rs_state pointer points to the start of what we'll
363 * copy to the guest, which in the case of a compat guest
364 * is the 32-bit field that the compiler thinks is padding.
365 */
366 rs_state = ((void *)rs_times) - times_ofs;
367
368 /*
369 * The update_bit is still directly in the guest memory,
370 * via one GPC or the other.
371 */
372 if (v->kvm->arch.xen.runstate_update_flag) {
373 if (user_len1 >= times_ofs + sizeof(uint64_t))
374 update_bit = gpc1->khva + times_ofs +
375 sizeof(uint64_t) - 1;
376 else
377 update_bit = gpc2->khva + times_ofs +
378 sizeof(uint64_t) - 1 - user_len1;
379 }
380
381#ifdef CONFIG_X86_64
382 /*
383 * Don't leak kernel memory through the padding in the 64-bit
384 * version of the struct.
385 */
386 memset(&rs, 0, offsetof(struct vcpu_runstate_info, state_entry_time));
387#endif
388 }
389
390 /*
391 * First, set the XEN_RUNSTATE_UPDATE bit in the top bit of the
392 * state_entry_time field, directly in the guest. We need to set
393 * that (and write-barrier) before writing to the rest of the
394 * structure, and clear it last. Just as Xen does, we address the
395 * single *byte* in which it resides because it might be in a
396 * different cache line to the rest of the 64-bit word, due to
397 * the (lack of) alignment constraints.
398 */
399 entry_time = vx->runstate_entry_time;
400 if (update_bit) {
401 entry_time |= XEN_RUNSTATE_UPDATE;
402 *update_bit = (vx->runstate_entry_time | XEN_RUNSTATE_UPDATE) >> 56;
403 smp_wmb();
404 }
405
406 /*
407 * Now assemble the actual structure, either on our kernel stack
408 * or directly in the guest according to how the rs_state and
409 * rs_times pointers were set up above.
410 */
411 *rs_state = vx->current_runstate;
412 rs_times[0] = entry_time;
413 memcpy(rs_times + 1, vx->runstate_times, sizeof(vx->runstate_times));
414
415 /* For the split case, we have to then copy it to the guest. */
416 if (user_len2) {
417 memcpy(gpc1->khva, rs_state, user_len1);
418 memcpy(gpc2->khva, ((void *)rs_state) + user_len1, user_len2);
419 }
420 smp_wmb();
421
422 /* Finally, clear the XEN_RUNSTATE_UPDATE bit. */
423 if (update_bit) {
424 entry_time &= ~XEN_RUNSTATE_UPDATE;
425 *update_bit = entry_time >> 56;
426 smp_wmb();
427 }
428
429 if (user_len2)
430 read_unlock(&gpc2->lock);
431
432 read_unlock_irqrestore(&gpc1->lock, flags);
433
434 mark_page_dirty_in_slot(v->kvm, gpc1->memslot, gpc1->gpa >> PAGE_SHIFT);
435 if (user_len2)
436 mark_page_dirty_in_slot(v->kvm, gpc2->memslot, gpc2->gpa >> PAGE_SHIFT);
437}
438
439void kvm_xen_update_runstate(struct kvm_vcpu *v, int state)
440{
441 struct kvm_vcpu_xen *vx = &v->arch.xen;
442 u64 now = get_kvmclock_ns(v->kvm);
443 u64 delta_ns = now - vx->runstate_entry_time;
444 u64 run_delay = current->sched_info.run_delay;
445
446 if (unlikely(!vx->runstate_entry_time))
447 vx->current_runstate = RUNSTATE_offline;
448
449 /*
450 * Time waiting for the scheduler isn't "stolen" if the
451 * vCPU wasn't running anyway.
452 */
453 if (vx->current_runstate == RUNSTATE_running) {
454 u64 steal_ns = run_delay - vx->last_steal;
455
456 delta_ns -= steal_ns;
457
458 vx->runstate_times[RUNSTATE_runnable] += steal_ns;
459 }
460 vx->last_steal = run_delay;
461
462 vx->runstate_times[vx->current_runstate] += delta_ns;
463 vx->current_runstate = state;
464 vx->runstate_entry_time = now;
465
466 if (vx->runstate_cache.active)
467 kvm_xen_update_runstate_guest(v, state == RUNSTATE_runnable);
468}
469
470static void kvm_xen_inject_vcpu_vector(struct kvm_vcpu *v)
471{
472 struct kvm_lapic_irq irq = { };
473 int r;
474
475 irq.dest_id = v->vcpu_id;
476 irq.vector = v->arch.xen.upcall_vector;
477 irq.dest_mode = APIC_DEST_PHYSICAL;
478 irq.shorthand = APIC_DEST_NOSHORT;
479 irq.delivery_mode = APIC_DM_FIXED;
480 irq.level = 1;
481
482 /* The fast version will always work for physical unicast */
483 WARN_ON_ONCE(!kvm_irq_delivery_to_apic_fast(v->kvm, NULL, &irq, &r, NULL));
484}
485
486/*
487 * On event channel delivery, the vcpu_info may not have been accessible.
488 * In that case, there are bits in vcpu->arch.xen.evtchn_pending_sel which
489 * need to be marked into the vcpu_info (and evtchn_upcall_pending set).
490 * Do so now that we can sleep in the context of the vCPU to bring the
491 * page in, and refresh the pfn cache for it.
492 */
493void kvm_xen_inject_pending_events(struct kvm_vcpu *v)
494{
495 unsigned long evtchn_pending_sel = READ_ONCE(v->arch.xen.evtchn_pending_sel);
496 struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache;
497 unsigned long flags;
498
499 if (!evtchn_pending_sel)
500 return;
501
502 /*
503 * Yes, this is an open-coded loop. But that's just what put_user()
504 * does anyway. Page it in and retry the instruction. We're just a
505 * little more honest about it.
506 */
507 read_lock_irqsave(&gpc->lock, flags);
508 while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
509 read_unlock_irqrestore(&gpc->lock, flags);
510
511 if (kvm_gpc_refresh(gpc, sizeof(struct vcpu_info)))
512 return;
513
514 read_lock_irqsave(&gpc->lock, flags);
515 }
516
517 /* Now gpc->khva is a valid kernel address for the vcpu_info */
518 if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
519 struct vcpu_info *vi = gpc->khva;
520
521 asm volatile(LOCK_PREFIX "orq %0, %1\n"
522 "notq %0\n"
523 LOCK_PREFIX "andq %0, %2\n"
524 : "=r" (evtchn_pending_sel),
525 "+m" (vi->evtchn_pending_sel),
526 "+m" (v->arch.xen.evtchn_pending_sel)
527 : "0" (evtchn_pending_sel));
528 WRITE_ONCE(vi->evtchn_upcall_pending, 1);
529 } else {
530 u32 evtchn_pending_sel32 = evtchn_pending_sel;
531 struct compat_vcpu_info *vi = gpc->khva;
532
533 asm volatile(LOCK_PREFIX "orl %0, %1\n"
534 "notl %0\n"
535 LOCK_PREFIX "andl %0, %2\n"
536 : "=r" (evtchn_pending_sel32),
537 "+m" (vi->evtchn_pending_sel),
538 "+m" (v->arch.xen.evtchn_pending_sel)
539 : "0" (evtchn_pending_sel32));
540 WRITE_ONCE(vi->evtchn_upcall_pending, 1);
541 }
542 read_unlock_irqrestore(&gpc->lock, flags);
543
544 /* For the per-vCPU lapic vector, deliver it as MSI. */
545 if (v->arch.xen.upcall_vector)
546 kvm_xen_inject_vcpu_vector(v);
547
548 mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT);
549}
550
551int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
552{
553 struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache;
554 unsigned long flags;
555 u8 rc = 0;
556
557 /*
558 * If the global upcall vector (HVMIRQ_callback_vector) is set and
559 * the vCPU's evtchn_upcall_pending flag is set, the IRQ is pending.
560 */
561
562 /* No need for compat handling here */
563 BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
564 offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
565 BUILD_BUG_ON(sizeof(rc) !=
566 sizeof_field(struct vcpu_info, evtchn_upcall_pending));
567 BUILD_BUG_ON(sizeof(rc) !=
568 sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending));
569
570 read_lock_irqsave(&gpc->lock, flags);
571 while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
572 read_unlock_irqrestore(&gpc->lock, flags);
573
574 /*
575 * This function gets called from kvm_vcpu_block() after setting the
576 * task to TASK_INTERRUPTIBLE, to see if it needs to wake immediately
577 * from a HLT. So we really mustn't sleep. If the page ended up absent
578 * at that point, just return 1 in order to trigger an immediate wake,
579 * and we'll end up getting called again from a context where we *can*
580 * fault in the page and wait for it.
581 */
582 if (in_atomic() || !task_is_running(current))
583 return 1;
584
585 if (kvm_gpc_refresh(gpc, sizeof(struct vcpu_info))) {
586 /*
587 * If this failed, userspace has screwed up the
588 * vcpu_info mapping. No interrupts for you.
589 */
590 return 0;
591 }
592 read_lock_irqsave(&gpc->lock, flags);
593 }
594
595 rc = ((struct vcpu_info *)gpc->khva)->evtchn_upcall_pending;
596 read_unlock_irqrestore(&gpc->lock, flags);
597 return rc;
598}
599
600int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
601{
602 int r = -ENOENT;
603
604
605 switch (data->type) {
606 case KVM_XEN_ATTR_TYPE_LONG_MODE:
607 if (!IS_ENABLED(CONFIG_64BIT) && data->u.long_mode) {
608 r = -EINVAL;
609 } else {
610 mutex_lock(&kvm->arch.xen.xen_lock);
611 kvm->arch.xen.long_mode = !!data->u.long_mode;
612 mutex_unlock(&kvm->arch.xen.xen_lock);
613 r = 0;
614 }
615 break;
616
617 case KVM_XEN_ATTR_TYPE_SHARED_INFO:
618 mutex_lock(&kvm->arch.xen.xen_lock);
619 r = kvm_xen_shared_info_init(kvm, data->u.shared_info.gfn);
620 mutex_unlock(&kvm->arch.xen.xen_lock);
621 break;
622
623 case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
624 if (data->u.vector && data->u.vector < 0x10)
625 r = -EINVAL;
626 else {
627 mutex_lock(&kvm->arch.xen.xen_lock);
628 kvm->arch.xen.upcall_vector = data->u.vector;
629 mutex_unlock(&kvm->arch.xen.xen_lock);
630 r = 0;
631 }
632 break;
633
634 case KVM_XEN_ATTR_TYPE_EVTCHN:
635 r = kvm_xen_setattr_evtchn(kvm, data);
636 break;
637
638 case KVM_XEN_ATTR_TYPE_XEN_VERSION:
639 mutex_lock(&kvm->arch.xen.xen_lock);
640 kvm->arch.xen.xen_version = data->u.xen_version;
641 mutex_unlock(&kvm->arch.xen.xen_lock);
642 r = 0;
643 break;
644
645 case KVM_XEN_ATTR_TYPE_RUNSTATE_UPDATE_FLAG:
646 if (!sched_info_on()) {
647 r = -EOPNOTSUPP;
648 break;
649 }
650 mutex_lock(&kvm->arch.xen.xen_lock);
651 kvm->arch.xen.runstate_update_flag = !!data->u.runstate_update_flag;
652 mutex_unlock(&kvm->arch.xen.xen_lock);
653 r = 0;
654 break;
655
656 default:
657 break;
658 }
659
660 return r;
661}
662
663int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
664{
665 int r = -ENOENT;
666
667 mutex_lock(&kvm->arch.xen.xen_lock);
668
669 switch (data->type) {
670 case KVM_XEN_ATTR_TYPE_LONG_MODE:
671 data->u.long_mode = kvm->arch.xen.long_mode;
672 r = 0;
673 break;
674
675 case KVM_XEN_ATTR_TYPE_SHARED_INFO:
676 if (kvm->arch.xen.shinfo_cache.active)
677 data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa);
678 else
679 data->u.shared_info.gfn = KVM_XEN_INVALID_GFN;
680 r = 0;
681 break;
682
683 case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
684 data->u.vector = kvm->arch.xen.upcall_vector;
685 r = 0;
686 break;
687
688 case KVM_XEN_ATTR_TYPE_XEN_VERSION:
689 data->u.xen_version = kvm->arch.xen.xen_version;
690 r = 0;
691 break;
692
693 case KVM_XEN_ATTR_TYPE_RUNSTATE_UPDATE_FLAG:
694 if (!sched_info_on()) {
695 r = -EOPNOTSUPP;
696 break;
697 }
698 data->u.runstate_update_flag = kvm->arch.xen.runstate_update_flag;
699 r = 0;
700 break;
701
702 default:
703 break;
704 }
705
706 mutex_unlock(&kvm->arch.xen.xen_lock);
707 return r;
708}
709
710int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
711{
712 int idx, r = -ENOENT;
713
714 mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
715 idx = srcu_read_lock(&vcpu->kvm->srcu);
716
717 switch (data->type) {
718 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
719 /* No compat necessary here. */
720 BUILD_BUG_ON(sizeof(struct vcpu_info) !=
721 sizeof(struct compat_vcpu_info));
722 BUILD_BUG_ON(offsetof(struct vcpu_info, time) !=
723 offsetof(struct compat_vcpu_info, time));
724
725 if (data->u.gpa == KVM_XEN_INVALID_GPA) {
726 kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_info_cache);
727 r = 0;
728 break;
729 }
730
731 r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache,
732 data->u.gpa, sizeof(struct vcpu_info));
733 if (!r)
734 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
735
736 break;
737
738 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
739 if (data->u.gpa == KVM_XEN_INVALID_GPA) {
740 kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_time_info_cache);
741 r = 0;
742 break;
743 }
744
745 r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_time_info_cache,
746 data->u.gpa,
747 sizeof(struct pvclock_vcpu_time_info));
748 if (!r)
749 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
750 break;
751
752 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR: {
753 size_t sz, sz1, sz2;
754
755 if (!sched_info_on()) {
756 r = -EOPNOTSUPP;
757 break;
758 }
759 if (data->u.gpa == KVM_XEN_INVALID_GPA) {
760 r = 0;
761 deactivate_out:
762 kvm_gpc_deactivate(&vcpu->arch.xen.runstate_cache);
763 kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
764 break;
765 }
766
767 /*
768 * If the guest switches to 64-bit mode after setting the runstate
769 * address, that's actually OK. kvm_xen_update_runstate_guest()
770 * will cope.
771 */
772 if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode)
773 sz = sizeof(struct vcpu_runstate_info);
774 else
775 sz = sizeof(struct compat_vcpu_runstate_info);
776
777 /* How much fits in the (first) page? */
778 sz1 = PAGE_SIZE - (data->u.gpa & ~PAGE_MASK);
779 r = kvm_gpc_activate(&vcpu->arch.xen.runstate_cache,
780 data->u.gpa, sz1);
781 if (r)
782 goto deactivate_out;
783
784 /* Either map the second page, or deactivate the second GPC */
785 if (sz1 >= sz) {
786 kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
787 } else {
788 sz2 = sz - sz1;
789 BUG_ON((data->u.gpa + sz1) & ~PAGE_MASK);
790 r = kvm_gpc_activate(&vcpu->arch.xen.runstate2_cache,
791 data->u.gpa + sz1, sz2);
792 if (r)
793 goto deactivate_out;
794 }
795
796 kvm_xen_update_runstate_guest(vcpu, false);
797 break;
798 }
799 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT:
800 if (!sched_info_on()) {
801 r = -EOPNOTSUPP;
802 break;
803 }
804 if (data->u.runstate.state > RUNSTATE_offline) {
805 r = -EINVAL;
806 break;
807 }
808
809 kvm_xen_update_runstate(vcpu, data->u.runstate.state);
810 r = 0;
811 break;
812
813 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA:
814 if (!sched_info_on()) {
815 r = -EOPNOTSUPP;
816 break;
817 }
818 if (data->u.runstate.state > RUNSTATE_offline) {
819 r = -EINVAL;
820 break;
821 }
822 if (data->u.runstate.state_entry_time !=
823 (data->u.runstate.time_running +
824 data->u.runstate.time_runnable +
825 data->u.runstate.time_blocked +
826 data->u.runstate.time_offline)) {
827 r = -EINVAL;
828 break;
829 }
830 if (get_kvmclock_ns(vcpu->kvm) <
831 data->u.runstate.state_entry_time) {
832 r = -EINVAL;
833 break;
834 }
835
836 vcpu->arch.xen.current_runstate = data->u.runstate.state;
837 vcpu->arch.xen.runstate_entry_time =
838 data->u.runstate.state_entry_time;
839 vcpu->arch.xen.runstate_times[RUNSTATE_running] =
840 data->u.runstate.time_running;
841 vcpu->arch.xen.runstate_times[RUNSTATE_runnable] =
842 data->u.runstate.time_runnable;
843 vcpu->arch.xen.runstate_times[RUNSTATE_blocked] =
844 data->u.runstate.time_blocked;
845 vcpu->arch.xen.runstate_times[RUNSTATE_offline] =
846 data->u.runstate.time_offline;
847 vcpu->arch.xen.last_steal = current->sched_info.run_delay;
848 r = 0;
849 break;
850
851 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST:
852 if (!sched_info_on()) {
853 r = -EOPNOTSUPP;
854 break;
855 }
856 if (data->u.runstate.state > RUNSTATE_offline &&
857 data->u.runstate.state != (u64)-1) {
858 r = -EINVAL;
859 break;
860 }
861 /* The adjustment must add up */
862 if (data->u.runstate.state_entry_time !=
863 (data->u.runstate.time_running +
864 data->u.runstate.time_runnable +
865 data->u.runstate.time_blocked +
866 data->u.runstate.time_offline)) {
867 r = -EINVAL;
868 break;
869 }
870
871 if (get_kvmclock_ns(vcpu->kvm) <
872 (vcpu->arch.xen.runstate_entry_time +
873 data->u.runstate.state_entry_time)) {
874 r = -EINVAL;
875 break;
876 }
877
878 vcpu->arch.xen.runstate_entry_time +=
879 data->u.runstate.state_entry_time;
880 vcpu->arch.xen.runstate_times[RUNSTATE_running] +=
881 data->u.runstate.time_running;
882 vcpu->arch.xen.runstate_times[RUNSTATE_runnable] +=
883 data->u.runstate.time_runnable;
884 vcpu->arch.xen.runstate_times[RUNSTATE_blocked] +=
885 data->u.runstate.time_blocked;
886 vcpu->arch.xen.runstate_times[RUNSTATE_offline] +=
887 data->u.runstate.time_offline;
888
889 if (data->u.runstate.state <= RUNSTATE_offline)
890 kvm_xen_update_runstate(vcpu, data->u.runstate.state);
891 else if (vcpu->arch.xen.runstate_cache.active)
892 kvm_xen_update_runstate_guest(vcpu, false);
893 r = 0;
894 break;
895
896 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID:
897 if (data->u.vcpu_id >= KVM_MAX_VCPUS)
898 r = -EINVAL;
899 else {
900 vcpu->arch.xen.vcpu_id = data->u.vcpu_id;
901 r = 0;
902 }
903 break;
904
905 case KVM_XEN_VCPU_ATTR_TYPE_TIMER:
906 if (data->u.timer.port &&
907 data->u.timer.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) {
908 r = -EINVAL;
909 break;
910 }
911
912 if (!vcpu->arch.xen.timer.function)
913 kvm_xen_init_timer(vcpu);
914
915 /* Stop the timer (if it's running) before changing the vector */
916 kvm_xen_stop_timer(vcpu);
917 vcpu->arch.xen.timer_virq = data->u.timer.port;
918
919 /* Start the timer if the new value has a valid vector+expiry. */
920 if (data->u.timer.port && data->u.timer.expires_ns)
921 kvm_xen_start_timer(vcpu, data->u.timer.expires_ns,
922 data->u.timer.expires_ns -
923 get_kvmclock_ns(vcpu->kvm));
924
925 r = 0;
926 break;
927
928 case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR:
929 if (data->u.vector && data->u.vector < 0x10)
930 r = -EINVAL;
931 else {
932 vcpu->arch.xen.upcall_vector = data->u.vector;
933 r = 0;
934 }
935 break;
936
937 default:
938 break;
939 }
940
941 srcu_read_unlock(&vcpu->kvm->srcu, idx);
942 mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
943 return r;
944}
945
946int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
947{
948 int r = -ENOENT;
949
950 mutex_lock(&vcpu->kvm->arch.xen.xen_lock);
951
952 switch (data->type) {
953 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
954 if (vcpu->arch.xen.vcpu_info_cache.active)
955 data->u.gpa = vcpu->arch.xen.vcpu_info_cache.gpa;
956 else
957 data->u.gpa = KVM_XEN_INVALID_GPA;
958 r = 0;
959 break;
960
961 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
962 if (vcpu->arch.xen.vcpu_time_info_cache.active)
963 data->u.gpa = vcpu->arch.xen.vcpu_time_info_cache.gpa;
964 else
965 data->u.gpa = KVM_XEN_INVALID_GPA;
966 r = 0;
967 break;
968
969 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR:
970 if (!sched_info_on()) {
971 r = -EOPNOTSUPP;
972 break;
973 }
974 if (vcpu->arch.xen.runstate_cache.active) {
975 data->u.gpa = vcpu->arch.xen.runstate_cache.gpa;
976 r = 0;
977 }
978 break;
979
980 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT:
981 if (!sched_info_on()) {
982 r = -EOPNOTSUPP;
983 break;
984 }
985 data->u.runstate.state = vcpu->arch.xen.current_runstate;
986 r = 0;
987 break;
988
989 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA:
990 if (!sched_info_on()) {
991 r = -EOPNOTSUPP;
992 break;
993 }
994 data->u.runstate.state = vcpu->arch.xen.current_runstate;
995 data->u.runstate.state_entry_time =
996 vcpu->arch.xen.runstate_entry_time;
997 data->u.runstate.time_running =
998 vcpu->arch.xen.runstate_times[RUNSTATE_running];
999 data->u.runstate.time_runnable =
1000 vcpu->arch.xen.runstate_times[RUNSTATE_runnable];
1001 data->u.runstate.time_blocked =
1002 vcpu->arch.xen.runstate_times[RUNSTATE_blocked];
1003 data->u.runstate.time_offline =
1004 vcpu->arch.xen.runstate_times[RUNSTATE_offline];
1005 r = 0;
1006 break;
1007
1008 case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST:
1009 r = -EINVAL;
1010 break;
1011
1012 case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID:
1013 data->u.vcpu_id = vcpu->arch.xen.vcpu_id;
1014 r = 0;
1015 break;
1016
1017 case KVM_XEN_VCPU_ATTR_TYPE_TIMER:
1018 data->u.timer.port = vcpu->arch.xen.timer_virq;
1019 data->u.timer.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL;
1020 data->u.timer.expires_ns = vcpu->arch.xen.timer_expires;
1021 r = 0;
1022 break;
1023
1024 case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR:
1025 data->u.vector = vcpu->arch.xen.upcall_vector;
1026 r = 0;
1027 break;
1028
1029 default:
1030 break;
1031 }
1032
1033 mutex_unlock(&vcpu->kvm->arch.xen.xen_lock);
1034 return r;
1035}
1036
1037int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data)
1038{
1039 struct kvm *kvm = vcpu->kvm;
1040 u32 page_num = data & ~PAGE_MASK;
1041 u64 page_addr = data & PAGE_MASK;
1042 bool lm = is_long_mode(vcpu);
1043
1044 /* Latch long_mode for shared_info pages etc. */
1045 vcpu->kvm->arch.xen.long_mode = lm;
1046
1047 /*
1048 * If Xen hypercall intercept is enabled, fill the hypercall
1049 * page with VMCALL/VMMCALL instructions since that's what
1050 * we catch. Else the VMM has provided the hypercall pages
1051 * with instructions of its own choosing, so use those.
1052 */
1053 if (kvm_xen_hypercall_enabled(kvm)) {
1054 u8 instructions[32];
1055 int i;
1056
1057 if (page_num)
1058 return 1;
1059
1060 /* mov imm32, %eax */
1061 instructions[0] = 0xb8;
1062
1063 /* vmcall / vmmcall */
1064 static_call(kvm_x86_patch_hypercall)(vcpu, instructions + 5);
1065
1066 /* ret */
1067 instructions[8] = 0xc3;
1068
1069 /* int3 to pad */
1070 memset(instructions + 9, 0xcc, sizeof(instructions) - 9);
1071
1072 for (i = 0; i < PAGE_SIZE / sizeof(instructions); i++) {
1073 *(u32 *)&instructions[1] = i;
1074 if (kvm_vcpu_write_guest(vcpu,
1075 page_addr + (i * sizeof(instructions)),
1076 instructions, sizeof(instructions)))
1077 return 1;
1078 }
1079 } else {
1080 /*
1081 * Note, truncation is a non-issue as 'lm' is guaranteed to be
1082 * false for a 32-bit kernel, i.e. when hva_t is only 4 bytes.
1083 */
1084 hva_t blob_addr = lm ? kvm->arch.xen_hvm_config.blob_addr_64
1085 : kvm->arch.xen_hvm_config.blob_addr_32;
1086 u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
1087 : kvm->arch.xen_hvm_config.blob_size_32;
1088 u8 *page;
1089 int ret;
1090
1091 if (page_num >= blob_size)
1092 return 1;
1093
1094 blob_addr += page_num * PAGE_SIZE;
1095
1096 page = memdup_user((u8 __user *)blob_addr, PAGE_SIZE);
1097 if (IS_ERR(page))
1098 return PTR_ERR(page);
1099
1100 ret = kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE);
1101 kfree(page);
1102 if (ret)
1103 return 1;
1104 }
1105 return 0;
1106}
1107
1108int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc)
1109{
1110 /* Only some feature flags need to be *enabled* by userspace */
1111 u32 permitted_flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
1112 KVM_XEN_HVM_CONFIG_EVTCHN_SEND;
1113
1114 if (xhc->flags & ~permitted_flags)
1115 return -EINVAL;
1116
1117 /*
1118 * With hypercall interception the kernel generates its own
1119 * hypercall page so it must not be provided.
1120 */
1121 if ((xhc->flags & KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) &&
1122 (xhc->blob_addr_32 || xhc->blob_addr_64 ||
1123 xhc->blob_size_32 || xhc->blob_size_64))
1124 return -EINVAL;
1125
1126 mutex_lock(&kvm->arch.xen.xen_lock);
1127
1128 if (xhc->msr && !kvm->arch.xen_hvm_config.msr)
1129 static_branch_inc(&kvm_xen_enabled.key);
1130 else if (!xhc->msr && kvm->arch.xen_hvm_config.msr)
1131 static_branch_slow_dec_deferred(&kvm_xen_enabled);
1132
1133 memcpy(&kvm->arch.xen_hvm_config, xhc, sizeof(*xhc));
1134
1135 mutex_unlock(&kvm->arch.xen.xen_lock);
1136 return 0;
1137}
1138
1139static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
1140{
1141 kvm_rax_write(vcpu, result);
1142 return kvm_skip_emulated_instruction(vcpu);
1143}
1144
1145static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
1146{
1147 struct kvm_run *run = vcpu->run;
1148
1149 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.xen.hypercall_rip)))
1150 return 1;
1151
1152 return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result);
1153}
1154
1155static inline int max_evtchn_port(struct kvm *kvm)
1156{
1157 if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode)
1158 return EVTCHN_2L_NR_CHANNELS;
1159 else
1160 return COMPAT_EVTCHN_2L_NR_CHANNELS;
1161}
1162
1163static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports,
1164 evtchn_port_t *ports)
1165{
1166 struct kvm *kvm = vcpu->kvm;
1167 struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1168 unsigned long *pending_bits;
1169 unsigned long flags;
1170 bool ret = true;
1171 int idx, i;
1172
1173 idx = srcu_read_lock(&kvm->srcu);
1174 read_lock_irqsave(&gpc->lock, flags);
1175 if (!kvm_gpc_check(gpc, PAGE_SIZE))
1176 goto out_rcu;
1177
1178 ret = false;
1179 if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1180 struct shared_info *shinfo = gpc->khva;
1181 pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1182 } else {
1183 struct compat_shared_info *shinfo = gpc->khva;
1184 pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1185 }
1186
1187 for (i = 0; i < nr_ports; i++) {
1188 if (test_bit(ports[i], pending_bits)) {
1189 ret = true;
1190 break;
1191 }
1192 }
1193
1194 out_rcu:
1195 read_unlock_irqrestore(&gpc->lock, flags);
1196 srcu_read_unlock(&kvm->srcu, idx);
1197
1198 return ret;
1199}
1200
1201static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
1202 u64 param, u64 *r)
1203{
1204 struct sched_poll sched_poll;
1205 evtchn_port_t port, *ports;
1206 struct x86_exception e;
1207 int i;
1208
1209 if (!lapic_in_kernel(vcpu) ||
1210 !(vcpu->kvm->arch.xen_hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND))
1211 return false;
1212
1213 if (IS_ENABLED(CONFIG_64BIT) && !longmode) {
1214 struct compat_sched_poll sp32;
1215
1216 /* Sanity check that the compat struct definition is correct */
1217 BUILD_BUG_ON(sizeof(sp32) != 16);
1218
1219 if (kvm_read_guest_virt(vcpu, param, &sp32, sizeof(sp32), &e)) {
1220 *r = -EFAULT;
1221 return true;
1222 }
1223
1224 /*
1225 * This is a 32-bit pointer to an array of evtchn_port_t which
1226 * are uint32_t, so once it's converted no further compat
1227 * handling is needed.
1228 */
1229 sched_poll.ports = (void *)(unsigned long)(sp32.ports);
1230 sched_poll.nr_ports = sp32.nr_ports;
1231 sched_poll.timeout = sp32.timeout;
1232 } else {
1233 if (kvm_read_guest_virt(vcpu, param, &sched_poll,
1234 sizeof(sched_poll), &e)) {
1235 *r = -EFAULT;
1236 return true;
1237 }
1238 }
1239
1240 if (unlikely(sched_poll.nr_ports > 1)) {
1241 /* Xen (unofficially) limits number of pollers to 128 */
1242 if (sched_poll.nr_ports > 128) {
1243 *r = -EINVAL;
1244 return true;
1245 }
1246
1247 ports = kmalloc_array(sched_poll.nr_ports,
1248 sizeof(*ports), GFP_KERNEL);
1249 if (!ports) {
1250 *r = -ENOMEM;
1251 return true;
1252 }
1253 } else
1254 ports = &port;
1255
1256 if (kvm_read_guest_virt(vcpu, (gva_t)sched_poll.ports, ports,
1257 sched_poll.nr_ports * sizeof(*ports), &e)) {
1258 *r = -EFAULT;
1259 return true;
1260 }
1261
1262 for (i = 0; i < sched_poll.nr_ports; i++) {
1263 if (ports[i] >= max_evtchn_port(vcpu->kvm)) {
1264 *r = -EINVAL;
1265 goto out;
1266 }
1267 }
1268
1269 if (sched_poll.nr_ports == 1)
1270 vcpu->arch.xen.poll_evtchn = port;
1271 else
1272 vcpu->arch.xen.poll_evtchn = -1;
1273
1274 set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
1275
1276 if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) {
1277 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
1278
1279 if (sched_poll.timeout)
1280 mod_timer(&vcpu->arch.xen.poll_timer,
1281 jiffies + nsecs_to_jiffies(sched_poll.timeout));
1282
1283 kvm_vcpu_halt(vcpu);
1284
1285 if (sched_poll.timeout)
1286 del_timer(&vcpu->arch.xen.poll_timer);
1287
1288 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
1289 }
1290
1291 vcpu->arch.xen.poll_evtchn = 0;
1292 *r = 0;
1293out:
1294 /* Really, this is only needed in case of timeout */
1295 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
1296
1297 if (unlikely(sched_poll.nr_ports > 1))
1298 kfree(ports);
1299 return true;
1300}
1301
1302static void cancel_evtchn_poll(struct timer_list *t)
1303{
1304 struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.xen.poll_timer);
1305
1306 kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1307 kvm_vcpu_kick(vcpu);
1308}
1309
1310static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode,
1311 int cmd, u64 param, u64 *r)
1312{
1313 switch (cmd) {
1314 case SCHEDOP_poll:
1315 if (kvm_xen_schedop_poll(vcpu, longmode, param, r))
1316 return true;
1317 fallthrough;
1318 case SCHEDOP_yield:
1319 kvm_vcpu_on_spin(vcpu, true);
1320 *r = 0;
1321 return true;
1322 default:
1323 break;
1324 }
1325
1326 return false;
1327}
1328
1329struct compat_vcpu_set_singleshot_timer {
1330 uint64_t timeout_abs_ns;
1331 uint32_t flags;
1332} __attribute__((packed));
1333
1334static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
1335 int vcpu_id, u64 param, u64 *r)
1336{
1337 struct vcpu_set_singleshot_timer oneshot;
1338 struct x86_exception e;
1339 s64 delta;
1340
1341 if (!kvm_xen_timer_enabled(vcpu))
1342 return false;
1343
1344 switch (cmd) {
1345 case VCPUOP_set_singleshot_timer:
1346 if (vcpu->arch.xen.vcpu_id != vcpu_id) {
1347 *r = -EINVAL;
1348 return true;
1349 }
1350
1351 /*
1352 * The only difference for 32-bit compat is the 4 bytes of
1353 * padding after the interesting part of the structure. So
1354 * for a faithful emulation of Xen we have to *try* to copy
1355 * the padding and return -EFAULT if we can't. Otherwise we
1356 * might as well just have copied the 12-byte 32-bit struct.
1357 */
1358 BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) !=
1359 offsetof(struct vcpu_set_singleshot_timer, timeout_abs_ns));
1360 BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) !=
1361 sizeof_field(struct vcpu_set_singleshot_timer, timeout_abs_ns));
1362 BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, flags) !=
1363 offsetof(struct vcpu_set_singleshot_timer, flags));
1364 BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) !=
1365 sizeof_field(struct vcpu_set_singleshot_timer, flags));
1366
1367 if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) :
1368 sizeof(struct compat_vcpu_set_singleshot_timer), &e)) {
1369 *r = -EFAULT;
1370 return true;
1371 }
1372
1373 delta = oneshot.timeout_abs_ns - get_kvmclock_ns(vcpu->kvm);
1374 if ((oneshot.flags & VCPU_SSHOTTMR_future) && delta < 0) {
1375 *r = -ETIME;
1376 return true;
1377 }
1378
1379 kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, delta);
1380 *r = 0;
1381 return true;
1382
1383 case VCPUOP_stop_singleshot_timer:
1384 if (vcpu->arch.xen.vcpu_id != vcpu_id) {
1385 *r = -EINVAL;
1386 return true;
1387 }
1388 kvm_xen_stop_timer(vcpu);
1389 *r = 0;
1390 return true;
1391 }
1392
1393 return false;
1394}
1395
1396static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout,
1397 u64 *r)
1398{
1399 if (!kvm_xen_timer_enabled(vcpu))
1400 return false;
1401
1402 if (timeout) {
1403 uint64_t guest_now = get_kvmclock_ns(vcpu->kvm);
1404 int64_t delta = timeout - guest_now;
1405
1406 /* Xen has a 'Linux workaround' in do_set_timer_op() which
1407 * checks for negative absolute timeout values (caused by
1408 * integer overflow), and for values about 13 days in the
1409 * future (2^50ns) which would be caused by jiffies
1410 * overflow. For those cases, it sets the timeout 100ms in
1411 * the future (not *too* soon, since if a guest really did
1412 * set a long timeout on purpose we don't want to keep
1413 * churning CPU time by waking it up).
1414 */
1415 if (unlikely((int64_t)timeout < 0 ||
1416 (delta > 0 && (uint32_t) (delta >> 50) != 0))) {
1417 delta = 100 * NSEC_PER_MSEC;
1418 timeout = guest_now + delta;
1419 }
1420
1421 kvm_xen_start_timer(vcpu, timeout, delta);
1422 } else {
1423 kvm_xen_stop_timer(vcpu);
1424 }
1425
1426 *r = 0;
1427 return true;
1428}
1429
1430int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
1431{
1432 bool longmode;
1433 u64 input, params[6], r = -ENOSYS;
1434 bool handled = false;
1435 u8 cpl;
1436
1437 input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX);
1438
1439 /* Hyper-V hypercalls get bit 31 set in EAX */
1440 if ((input & 0x80000000) &&
1441 kvm_hv_hypercall_enabled(vcpu))
1442 return kvm_hv_hypercall(vcpu);
1443
1444 longmode = is_64_bit_hypercall(vcpu);
1445 if (!longmode) {
1446 params[0] = (u32)kvm_rbx_read(vcpu);
1447 params[1] = (u32)kvm_rcx_read(vcpu);
1448 params[2] = (u32)kvm_rdx_read(vcpu);
1449 params[3] = (u32)kvm_rsi_read(vcpu);
1450 params[4] = (u32)kvm_rdi_read(vcpu);
1451 params[5] = (u32)kvm_rbp_read(vcpu);
1452 }
1453#ifdef CONFIG_X86_64
1454 else {
1455 params[0] = (u64)kvm_rdi_read(vcpu);
1456 params[1] = (u64)kvm_rsi_read(vcpu);
1457 params[2] = (u64)kvm_rdx_read(vcpu);
1458 params[3] = (u64)kvm_r10_read(vcpu);
1459 params[4] = (u64)kvm_r8_read(vcpu);
1460 params[5] = (u64)kvm_r9_read(vcpu);
1461 }
1462#endif
1463 cpl = static_call(kvm_x86_get_cpl)(vcpu);
1464 trace_kvm_xen_hypercall(cpl, input, params[0], params[1], params[2],
1465 params[3], params[4], params[5]);
1466
1467 /*
1468 * Only allow hypercall acceleration for CPL0. The rare hypercalls that
1469 * are permitted in guest userspace can be handled by the VMM.
1470 */
1471 if (unlikely(cpl > 0))
1472 goto handle_in_userspace;
1473
1474 switch (input) {
1475 case __HYPERVISOR_xen_version:
1476 if (params[0] == XENVER_version && vcpu->kvm->arch.xen.xen_version) {
1477 r = vcpu->kvm->arch.xen.xen_version;
1478 handled = true;
1479 }
1480 break;
1481 case __HYPERVISOR_event_channel_op:
1482 if (params[0] == EVTCHNOP_send)
1483 handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r);
1484 break;
1485 case __HYPERVISOR_sched_op:
1486 handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0],
1487 params[1], &r);
1488 break;
1489 case __HYPERVISOR_vcpu_op:
1490 handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1],
1491 params[2], &r);
1492 break;
1493 case __HYPERVISOR_set_timer_op: {
1494 u64 timeout = params[0];
1495 /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */
1496 if (!longmode)
1497 timeout |= params[1] << 32;
1498 handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r);
1499 break;
1500 }
1501 default:
1502 break;
1503 }
1504
1505 if (handled)
1506 return kvm_xen_hypercall_set_result(vcpu, r);
1507
1508handle_in_userspace:
1509 vcpu->run->exit_reason = KVM_EXIT_XEN;
1510 vcpu->run->xen.type = KVM_EXIT_XEN_HCALL;
1511 vcpu->run->xen.u.hcall.longmode = longmode;
1512 vcpu->run->xen.u.hcall.cpl = cpl;
1513 vcpu->run->xen.u.hcall.input = input;
1514 vcpu->run->xen.u.hcall.params[0] = params[0];
1515 vcpu->run->xen.u.hcall.params[1] = params[1];
1516 vcpu->run->xen.u.hcall.params[2] = params[2];
1517 vcpu->run->xen.u.hcall.params[3] = params[3];
1518 vcpu->run->xen.u.hcall.params[4] = params[4];
1519 vcpu->run->xen.u.hcall.params[5] = params[5];
1520 vcpu->arch.xen.hypercall_rip = kvm_get_linear_rip(vcpu);
1521 vcpu->arch.complete_userspace_io =
1522 kvm_xen_hypercall_complete_userspace;
1523
1524 return 0;
1525}
1526
1527static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port)
1528{
1529 int poll_evtchn = vcpu->arch.xen.poll_evtchn;
1530
1531 if ((poll_evtchn == port || poll_evtchn == -1) &&
1532 test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) {
1533 kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1534 kvm_vcpu_kick(vcpu);
1535 }
1536}
1537
1538/*
1539 * The return value from this function is propagated to kvm_set_irq() API,
1540 * so it returns:
1541 * < 0 Interrupt was ignored (masked or not delivered for other reasons)
1542 * = 0 Interrupt was coalesced (previous irq is still pending)
1543 * > 0 Number of CPUs interrupt was delivered to
1544 *
1545 * It is also called directly from kvm_arch_set_irq_inatomic(), where the
1546 * only check on its return value is a comparison with -EWOULDBLOCK'.
1547 */
1548int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
1549{
1550 struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1551 struct kvm_vcpu *vcpu;
1552 unsigned long *pending_bits, *mask_bits;
1553 unsigned long flags;
1554 int port_word_bit;
1555 bool kick_vcpu = false;
1556 int vcpu_idx, idx, rc;
1557
1558 vcpu_idx = READ_ONCE(xe->vcpu_idx);
1559 if (vcpu_idx >= 0)
1560 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1561 else {
1562 vcpu = kvm_get_vcpu_by_id(kvm, xe->vcpu_id);
1563 if (!vcpu)
1564 return -EINVAL;
1565 WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx);
1566 }
1567
1568 if (!vcpu->arch.xen.vcpu_info_cache.active)
1569 return -EINVAL;
1570
1571 if (xe->port >= max_evtchn_port(kvm))
1572 return -EINVAL;
1573
1574 rc = -EWOULDBLOCK;
1575
1576 idx = srcu_read_lock(&kvm->srcu);
1577
1578 read_lock_irqsave(&gpc->lock, flags);
1579 if (!kvm_gpc_check(gpc, PAGE_SIZE))
1580 goto out_rcu;
1581
1582 if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1583 struct shared_info *shinfo = gpc->khva;
1584 pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1585 mask_bits = (unsigned long *)&shinfo->evtchn_mask;
1586 port_word_bit = xe->port / 64;
1587 } else {
1588 struct compat_shared_info *shinfo = gpc->khva;
1589 pending_bits = (unsigned long *)&shinfo->evtchn_pending;
1590 mask_bits = (unsigned long *)&shinfo->evtchn_mask;
1591 port_word_bit = xe->port / 32;
1592 }
1593
1594 /*
1595 * If this port wasn't already set, and if it isn't masked, then
1596 * we try to set the corresponding bit in the in-kernel shadow of
1597 * evtchn_pending_sel for the target vCPU. And if *that* wasn't
1598 * already set, then we kick the vCPU in question to write to the
1599 * *real* evtchn_pending_sel in its own guest vcpu_info struct.
1600 */
1601 if (test_and_set_bit(xe->port, pending_bits)) {
1602 rc = 0; /* It was already raised */
1603 } else if (test_bit(xe->port, mask_bits)) {
1604 rc = -ENOTCONN; /* Masked */
1605 kvm_xen_check_poller(vcpu, xe->port);
1606 } else {
1607 rc = 1; /* Delivered to the bitmap in shared_info. */
1608 /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */
1609 read_unlock_irqrestore(&gpc->lock, flags);
1610 gpc = &vcpu->arch.xen.vcpu_info_cache;
1611
1612 read_lock_irqsave(&gpc->lock, flags);
1613 if (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
1614 /*
1615 * Could not access the vcpu_info. Set the bit in-kernel
1616 * and prod the vCPU to deliver it for itself.
1617 */
1618 if (!test_and_set_bit(port_word_bit, &vcpu->arch.xen.evtchn_pending_sel))
1619 kick_vcpu = true;
1620 goto out_rcu;
1621 }
1622
1623 if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
1624 struct vcpu_info *vcpu_info = gpc->khva;
1625 if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) {
1626 WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
1627 kick_vcpu = true;
1628 }
1629 } else {
1630 struct compat_vcpu_info *vcpu_info = gpc->khva;
1631 if (!test_and_set_bit(port_word_bit,
1632 (unsigned long *)&vcpu_info->evtchn_pending_sel)) {
1633 WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
1634 kick_vcpu = true;
1635 }
1636 }
1637
1638 /* For the per-vCPU lapic vector, deliver it as MSI. */
1639 if (kick_vcpu && vcpu->arch.xen.upcall_vector) {
1640 kvm_xen_inject_vcpu_vector(vcpu);
1641 kick_vcpu = false;
1642 }
1643 }
1644
1645 out_rcu:
1646 read_unlock_irqrestore(&gpc->lock, flags);
1647 srcu_read_unlock(&kvm->srcu, idx);
1648
1649 if (kick_vcpu) {
1650 kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1651 kvm_vcpu_kick(vcpu);
1652 }
1653
1654 return rc;
1655}
1656
1657static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm)
1658{
1659 bool mm_borrowed = false;
1660 int rc;
1661
1662 rc = kvm_xen_set_evtchn_fast(xe, kvm);
1663 if (rc != -EWOULDBLOCK)
1664 return rc;
1665
1666 if (current->mm != kvm->mm) {
1667 /*
1668 * If not on a thread which already belongs to this KVM,
1669 * we'd better be in the irqfd workqueue.
1670 */
1671 if (WARN_ON_ONCE(current->mm))
1672 return -EINVAL;
1673
1674 kthread_use_mm(kvm->mm);
1675 mm_borrowed = true;
1676 }
1677
1678 mutex_lock(&kvm->arch.xen.xen_lock);
1679
1680 /*
1681 * It is theoretically possible for the page to be unmapped
1682 * and the MMU notifier to invalidate the shared_info before
1683 * we even get to use it. In that case, this looks like an
1684 * infinite loop. It was tempting to do it via the userspace
1685 * HVA instead... but that just *hides* the fact that it's
1686 * an infinite loop, because if a fault occurs and it waits
1687 * for the page to come back, it can *still* immediately
1688 * fault and have to wait again, repeatedly.
1689 *
1690 * Conversely, the page could also have been reinstated by
1691 * another thread before we even obtain the mutex above, so
1692 * check again *first* before remapping it.
1693 */
1694 do {
1695 struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
1696 int idx;
1697
1698 rc = kvm_xen_set_evtchn_fast(xe, kvm);
1699 if (rc != -EWOULDBLOCK)
1700 break;
1701
1702 idx = srcu_read_lock(&kvm->srcu);
1703 rc = kvm_gpc_refresh(gpc, PAGE_SIZE);
1704 srcu_read_unlock(&kvm->srcu, idx);
1705 } while(!rc);
1706
1707 mutex_unlock(&kvm->arch.xen.xen_lock);
1708
1709 if (mm_borrowed)
1710 kthread_unuse_mm(kvm->mm);
1711
1712 return rc;
1713}
1714
1715/* This is the version called from kvm_set_irq() as the .set function */
1716static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
1717 int irq_source_id, int level, bool line_status)
1718{
1719 if (!level)
1720 return -EINVAL;
1721
1722 return kvm_xen_set_evtchn(&e->xen_evtchn, kvm);
1723}
1724
1725/*
1726 * Set up an event channel interrupt from the KVM IRQ routing table.
1727 * Used for e.g. PIRQ from passed through physical devices.
1728 */
1729int kvm_xen_setup_evtchn(struct kvm *kvm,
1730 struct kvm_kernel_irq_routing_entry *e,
1731 const struct kvm_irq_routing_entry *ue)
1732
1733{
1734 struct kvm_vcpu *vcpu;
1735
1736 if (ue->u.xen_evtchn.port >= max_evtchn_port(kvm))
1737 return -EINVAL;
1738
1739 /* We only support 2 level event channels for now */
1740 if (ue->u.xen_evtchn.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1741 return -EINVAL;
1742
1743 /*
1744 * Xen gives us interesting mappings from vCPU index to APIC ID,
1745 * which means kvm_get_vcpu_by_id() has to iterate over all vCPUs
1746 * to find it. Do that once at setup time, instead of every time.
1747 * But beware that on live update / live migration, the routing
1748 * table might be reinstated before the vCPU threads have finished
1749 * recreating their vCPUs.
1750 */
1751 vcpu = kvm_get_vcpu_by_id(kvm, ue->u.xen_evtchn.vcpu);
1752 if (vcpu)
1753 e->xen_evtchn.vcpu_idx = vcpu->vcpu_idx;
1754 else
1755 e->xen_evtchn.vcpu_idx = -1;
1756
1757 e->xen_evtchn.port = ue->u.xen_evtchn.port;
1758 e->xen_evtchn.vcpu_id = ue->u.xen_evtchn.vcpu;
1759 e->xen_evtchn.priority = ue->u.xen_evtchn.priority;
1760 e->set = evtchn_set_fn;
1761
1762 return 0;
1763}
1764
1765/*
1766 * Explicit event sending from userspace with KVM_XEN_HVM_EVTCHN_SEND ioctl.
1767 */
1768int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *uxe)
1769{
1770 struct kvm_xen_evtchn e;
1771 int ret;
1772
1773 if (!uxe->port || uxe->port >= max_evtchn_port(kvm))
1774 return -EINVAL;
1775
1776 /* We only support 2 level event channels for now */
1777 if (uxe->priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1778 return -EINVAL;
1779
1780 e.port = uxe->port;
1781 e.vcpu_id = uxe->vcpu;
1782 e.vcpu_idx = -1;
1783 e.priority = uxe->priority;
1784
1785 ret = kvm_xen_set_evtchn(&e, kvm);
1786
1787 /*
1788 * None of that 'return 1 if it actually got delivered' nonsense.
1789 * We don't care if it was masked (-ENOTCONN) either.
1790 */
1791 if (ret > 0 || ret == -ENOTCONN)
1792 ret = 0;
1793
1794 return ret;
1795}
1796
1797/*
1798 * Support for *outbound* event channel events via the EVTCHNOP_send hypercall.
1799 */
1800struct evtchnfd {
1801 u32 send_port;
1802 u32 type;
1803 union {
1804 struct kvm_xen_evtchn port;
1805 struct {
1806 u32 port; /* zero */
1807 struct eventfd_ctx *ctx;
1808 } eventfd;
1809 } deliver;
1810};
1811
1812/*
1813 * Update target vCPU or priority for a registered sending channel.
1814 */
1815static int kvm_xen_eventfd_update(struct kvm *kvm,
1816 struct kvm_xen_hvm_attr *data)
1817{
1818 u32 port = data->u.evtchn.send_port;
1819 struct evtchnfd *evtchnfd;
1820 int ret;
1821
1822 /* Protect writes to evtchnfd as well as the idr lookup. */
1823 mutex_lock(&kvm->arch.xen.xen_lock);
1824 evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port);
1825
1826 ret = -ENOENT;
1827 if (!evtchnfd)
1828 goto out_unlock;
1829
1830 /* For an UPDATE, nothing may change except the priority/vcpu */
1831 ret = -EINVAL;
1832 if (evtchnfd->type != data->u.evtchn.type)
1833 goto out_unlock;
1834
1835 /*
1836 * Port cannot change, and if it's zero that was an eventfd
1837 * which can't be changed either.
1838 */
1839 if (!evtchnfd->deliver.port.port ||
1840 evtchnfd->deliver.port.port != data->u.evtchn.deliver.port.port)
1841 goto out_unlock;
1842
1843 /* We only support 2 level event channels for now */
1844 if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1845 goto out_unlock;
1846
1847 evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority;
1848 if (evtchnfd->deliver.port.vcpu_id != data->u.evtchn.deliver.port.vcpu) {
1849 evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu;
1850 evtchnfd->deliver.port.vcpu_idx = -1;
1851 }
1852 ret = 0;
1853out_unlock:
1854 mutex_unlock(&kvm->arch.xen.xen_lock);
1855 return ret;
1856}
1857
1858/*
1859 * Configure the target (eventfd or local port delivery) for sending on
1860 * a given event channel.
1861 */
1862static int kvm_xen_eventfd_assign(struct kvm *kvm,
1863 struct kvm_xen_hvm_attr *data)
1864{
1865 u32 port = data->u.evtchn.send_port;
1866 struct eventfd_ctx *eventfd = NULL;
1867 struct evtchnfd *evtchnfd;
1868 int ret = -EINVAL;
1869
1870 evtchnfd = kzalloc(sizeof(struct evtchnfd), GFP_KERNEL);
1871 if (!evtchnfd)
1872 return -ENOMEM;
1873
1874 switch(data->u.evtchn.type) {
1875 case EVTCHNSTAT_ipi:
1876 /* IPI must map back to the same port# */
1877 if (data->u.evtchn.deliver.port.port != data->u.evtchn.send_port)
1878 goto out_noeventfd; /* -EINVAL */
1879 break;
1880
1881 case EVTCHNSTAT_interdomain:
1882 if (data->u.evtchn.deliver.port.port) {
1883 if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm))
1884 goto out_noeventfd; /* -EINVAL */
1885 } else {
1886 eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd);
1887 if (IS_ERR(eventfd)) {
1888 ret = PTR_ERR(eventfd);
1889 goto out_noeventfd;
1890 }
1891 }
1892 break;
1893
1894 case EVTCHNSTAT_virq:
1895 case EVTCHNSTAT_closed:
1896 case EVTCHNSTAT_unbound:
1897 case EVTCHNSTAT_pirq:
1898 default: /* Unknown event channel type */
1899 goto out; /* -EINVAL */
1900 }
1901
1902 evtchnfd->send_port = data->u.evtchn.send_port;
1903 evtchnfd->type = data->u.evtchn.type;
1904 if (eventfd) {
1905 evtchnfd->deliver.eventfd.ctx = eventfd;
1906 } else {
1907 /* We only support 2 level event channels for now */
1908 if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL)
1909 goto out; /* -EINVAL; */
1910
1911 evtchnfd->deliver.port.port = data->u.evtchn.deliver.port.port;
1912 evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu;
1913 evtchnfd->deliver.port.vcpu_idx = -1;
1914 evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority;
1915 }
1916
1917 mutex_lock(&kvm->arch.xen.xen_lock);
1918 ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1,
1919 GFP_KERNEL);
1920 mutex_unlock(&kvm->arch.xen.xen_lock);
1921 if (ret >= 0)
1922 return 0;
1923
1924 if (ret == -ENOSPC)
1925 ret = -EEXIST;
1926out:
1927 if (eventfd)
1928 eventfd_ctx_put(eventfd);
1929out_noeventfd:
1930 kfree(evtchnfd);
1931 return ret;
1932}
1933
1934static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port)
1935{
1936 struct evtchnfd *evtchnfd;
1937
1938 mutex_lock(&kvm->arch.xen.xen_lock);
1939 evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port);
1940 mutex_unlock(&kvm->arch.xen.xen_lock);
1941
1942 if (!evtchnfd)
1943 return -ENOENT;
1944
1945 synchronize_srcu(&kvm->srcu);
1946 if (!evtchnfd->deliver.port.port)
1947 eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
1948 kfree(evtchnfd);
1949 return 0;
1950}
1951
1952static int kvm_xen_eventfd_reset(struct kvm *kvm)
1953{
1954 struct evtchnfd *evtchnfd, **all_evtchnfds;
1955 int i;
1956 int n = 0;
1957
1958 mutex_lock(&kvm->arch.xen.xen_lock);
1959
1960 /*
1961 * Because synchronize_srcu() cannot be called inside the
1962 * critical section, first collect all the evtchnfd objects
1963 * in an array as they are removed from evtchn_ports.
1964 */
1965 idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i)
1966 n++;
1967
1968 all_evtchnfds = kmalloc_array(n, sizeof(struct evtchnfd *), GFP_KERNEL);
1969 if (!all_evtchnfds) {
1970 mutex_unlock(&kvm->arch.xen.xen_lock);
1971 return -ENOMEM;
1972 }
1973
1974 n = 0;
1975 idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
1976 all_evtchnfds[n++] = evtchnfd;
1977 idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port);
1978 }
1979 mutex_unlock(&kvm->arch.xen.xen_lock);
1980
1981 synchronize_srcu(&kvm->srcu);
1982
1983 while (n--) {
1984 evtchnfd = all_evtchnfds[n];
1985 if (!evtchnfd->deliver.port.port)
1986 eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
1987 kfree(evtchnfd);
1988 }
1989 kfree(all_evtchnfds);
1990
1991 return 0;
1992}
1993
1994static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data)
1995{
1996 u32 port = data->u.evtchn.send_port;
1997
1998 if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET)
1999 return kvm_xen_eventfd_reset(kvm);
2000
2001 if (!port || port >= max_evtchn_port(kvm))
2002 return -EINVAL;
2003
2004 if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN)
2005 return kvm_xen_eventfd_deassign(kvm, port);
2006 if (data->u.evtchn.flags == KVM_XEN_EVTCHN_UPDATE)
2007 return kvm_xen_eventfd_update(kvm, data);
2008 if (data->u.evtchn.flags)
2009 return -EINVAL;
2010
2011 return kvm_xen_eventfd_assign(kvm, data);
2012}
2013
2014static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r)
2015{
2016 struct evtchnfd *evtchnfd;
2017 struct evtchn_send send;
2018 struct x86_exception e;
2019
2020 /* Sanity check: this structure is the same for 32-bit and 64-bit */
2021 BUILD_BUG_ON(sizeof(send) != 4);
2022 if (kvm_read_guest_virt(vcpu, param, &send, sizeof(send), &e)) {
2023 *r = -EFAULT;
2024 return true;
2025 }
2026
2027 /*
2028 * evtchnfd is protected by kvm->srcu; the idr lookup instead
2029 * is protected by RCU.
2030 */
2031 rcu_read_lock();
2032 evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port);
2033 rcu_read_unlock();
2034 if (!evtchnfd)
2035 return false;
2036
2037 if (evtchnfd->deliver.port.port) {
2038 int ret = kvm_xen_set_evtchn(&evtchnfd->deliver.port, vcpu->kvm);
2039 if (ret < 0 && ret != -ENOTCONN)
2040 return false;
2041 } else {
2042 eventfd_signal(evtchnfd->deliver.eventfd.ctx, 1);
2043 }
2044
2045 *r = 0;
2046 return true;
2047}
2048
2049void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu)
2050{
2051 vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx;
2052 vcpu->arch.xen.poll_evtchn = 0;
2053
2054 timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0);
2055
2056 kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm, NULL,
2057 KVM_HOST_USES_PFN);
2058 kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm, NULL,
2059 KVM_HOST_USES_PFN);
2060 kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache, vcpu->kvm, NULL,
2061 KVM_HOST_USES_PFN);
2062 kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache, vcpu->kvm, NULL,
2063 KVM_HOST_USES_PFN);
2064}
2065
2066void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu)
2067{
2068 if (kvm_xen_timer_enabled(vcpu))
2069 kvm_xen_stop_timer(vcpu);
2070
2071 kvm_gpc_deactivate(&vcpu->arch.xen.runstate_cache);
2072 kvm_gpc_deactivate(&vcpu->arch.xen.runstate2_cache);
2073 kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_info_cache);
2074 kvm_gpc_deactivate(&vcpu->arch.xen.vcpu_time_info_cache);
2075
2076 del_timer_sync(&vcpu->arch.xen.poll_timer);
2077}
2078
2079void kvm_xen_init_vm(struct kvm *kvm)
2080{
2081 mutex_init(&kvm->arch.xen.xen_lock);
2082 idr_init(&kvm->arch.xen.evtchn_ports);
2083 kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm, NULL, KVM_HOST_USES_PFN);
2084}
2085
2086void kvm_xen_destroy_vm(struct kvm *kvm)
2087{
2088 struct evtchnfd *evtchnfd;
2089 int i;
2090
2091 kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
2092
2093 idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
2094 if (!evtchnfd->deliver.port.port)
2095 eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
2096 kfree(evtchnfd);
2097 }
2098 idr_destroy(&kvm->arch.xen.evtchn_ports);
2099
2100 if (kvm->arch.xen_hvm_config.msr)
2101 static_branch_slow_dec_deferred(&kvm_xen_enabled);
2102}