Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine (KVM) Hypervisor
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
7 *
8 * Authors:
9 * Avi Kivity <avi@qumranet.com>
10 * Yaniv Kamay <yaniv@qumranet.com>
11 */
12
13#include <kvm/iodev.h>
14
15#include <linux/kvm_host.h>
16#include <linux/kvm.h>
17#include <linux/module.h>
18#include <linux/errno.h>
19#include <linux/percpu.h>
20#include <linux/mm.h>
21#include <linux/miscdevice.h>
22#include <linux/vmalloc.h>
23#include <linux/reboot.h>
24#include <linux/debugfs.h>
25#include <linux/highmem.h>
26#include <linux/file.h>
27#include <linux/syscore_ops.h>
28#include <linux/cpu.h>
29#include <linux/sched/signal.h>
30#include <linux/sched/mm.h>
31#include <linux/sched/stat.h>
32#include <linux/cpumask.h>
33#include <linux/smp.h>
34#include <linux/anon_inodes.h>
35#include <linux/profile.h>
36#include <linux/kvm_para.h>
37#include <linux/pagemap.h>
38#include <linux/mman.h>
39#include <linux/swap.h>
40#include <linux/bitops.h>
41#include <linux/spinlock.h>
42#include <linux/compat.h>
43#include <linux/srcu.h>
44#include <linux/hugetlb.h>
45#include <linux/slab.h>
46#include <linux/sort.h>
47#include <linux/bsearch.h>
48#include <linux/io.h>
49#include <linux/lockdep.h>
50#include <linux/kthread.h>
51#include <linux/suspend.h>
52
53#include <asm/processor.h>
54#include <asm/ioctl.h>
55#include <linux/uaccess.h>
56
57#include "coalesced_mmio.h"
58#include "async_pf.h"
59#include "kvm_mm.h"
60#include "vfio.h"
61
62#include <trace/events/ipi.h>
63
64#define CREATE_TRACE_POINTS
65#include <trace/events/kvm.h>
66
67#include <linux/kvm_dirty_ring.h>
68
69
70/* Worst case buffer size needed for holding an integer. */
71#define ITOA_MAX_LEN 12
72
73MODULE_AUTHOR("Qumranet");
74MODULE_DESCRIPTION("Kernel-based Virtual Machine (KVM) Hypervisor");
75MODULE_LICENSE("GPL");
76
77/* Architectures should define their poll value according to the halt latency */
78unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
79module_param(halt_poll_ns, uint, 0644);
80EXPORT_SYMBOL_GPL(halt_poll_ns);
81
82/* Default doubles per-vcpu halt_poll_ns. */
83unsigned int halt_poll_ns_grow = 2;
84module_param(halt_poll_ns_grow, uint, 0644);
85EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
86
87/* The start value to grow halt_poll_ns from */
88unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
89module_param(halt_poll_ns_grow_start, uint, 0644);
90EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
91
92/* Default halves per-vcpu halt_poll_ns. */
93unsigned int halt_poll_ns_shrink = 2;
94module_param(halt_poll_ns_shrink, uint, 0644);
95EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
96
97/*
98 * Allow direct access (from KVM or the CPU) without MMU notifier protection
99 * to unpinned pages.
100 */
101static bool allow_unsafe_mappings;
102module_param(allow_unsafe_mappings, bool, 0444);
103
104/*
105 * Ordering of locks:
106 *
107 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
108 */
109
110DEFINE_MUTEX(kvm_lock);
111LIST_HEAD(vm_list);
112
113static struct kmem_cache *kvm_vcpu_cache;
114
115static __read_mostly struct preempt_ops kvm_preempt_ops;
116static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);
117
118static struct dentry *kvm_debugfs_dir;
119
120static const struct file_operations stat_fops_per_vm;
121
122static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
123 unsigned long arg);
124#ifdef CONFIG_KVM_COMPAT
125static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
126 unsigned long arg);
127#define KVM_COMPAT(c) .compat_ioctl = (c)
128#else
129/*
130 * For architectures that don't implement a compat infrastructure,
131 * adopt a double line of defense:
132 * - Prevent a compat task from opening /dev/kvm
133 * - If the open has been done by a 64bit task, and the KVM fd
134 * passed to a compat task, let the ioctls fail.
135 */
136static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
137 unsigned long arg) { return -EINVAL; }
138
139static int kvm_no_compat_open(struct inode *inode, struct file *file)
140{
141 return is_compat_task() ? -ENODEV : 0;
142}
143#define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \
144 .open = kvm_no_compat_open
145#endif
146static int kvm_enable_virtualization(void);
147static void kvm_disable_virtualization(void);
148
149static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
150
151#define KVM_EVENT_CREATE_VM 0
152#define KVM_EVENT_DESTROY_VM 1
153static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
154static unsigned long long kvm_createvm_count;
155static unsigned long long kvm_active_vms;
156
157static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask);
158
159__weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
160{
161}
162
163/*
164 * Switches to specified vcpu, until a matching vcpu_put()
165 */
166void vcpu_load(struct kvm_vcpu *vcpu)
167{
168 int cpu = get_cpu();
169
170 __this_cpu_write(kvm_running_vcpu, vcpu);
171 preempt_notifier_register(&vcpu->preempt_notifier);
172 kvm_arch_vcpu_load(vcpu, cpu);
173 put_cpu();
174}
175EXPORT_SYMBOL_GPL(vcpu_load);
176
177void vcpu_put(struct kvm_vcpu *vcpu)
178{
179 preempt_disable();
180 kvm_arch_vcpu_put(vcpu);
181 preempt_notifier_unregister(&vcpu->preempt_notifier);
182 __this_cpu_write(kvm_running_vcpu, NULL);
183 preempt_enable();
184}
185EXPORT_SYMBOL_GPL(vcpu_put);
186
187/* TODO: merge with kvm_arch_vcpu_should_kick */
188static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
189{
190 int mode = kvm_vcpu_exiting_guest_mode(vcpu);
191
192 /*
193 * We need to wait for the VCPU to reenable interrupts and get out of
194 * READING_SHADOW_PAGE_TABLES mode.
195 */
196 if (req & KVM_REQUEST_WAIT)
197 return mode != OUTSIDE_GUEST_MODE;
198
199 /*
200 * Need to kick a running VCPU, but otherwise there is nothing to do.
201 */
202 return mode == IN_GUEST_MODE;
203}
204
205static void ack_kick(void *_completed)
206{
207}
208
209static inline bool kvm_kick_many_cpus(struct cpumask *cpus, bool wait)
210{
211 if (cpumask_empty(cpus))
212 return false;
213
214 smp_call_function_many(cpus, ack_kick, NULL, wait);
215 return true;
216}
217
218static void kvm_make_vcpu_request(struct kvm_vcpu *vcpu, unsigned int req,
219 struct cpumask *tmp, int current_cpu)
220{
221 int cpu;
222
223 if (likely(!(req & KVM_REQUEST_NO_ACTION)))
224 __kvm_make_request(req, vcpu);
225
226 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
227 return;
228
229 /*
230 * Note, the vCPU could get migrated to a different pCPU at any point
231 * after kvm_request_needs_ipi(), which could result in sending an IPI
232 * to the previous pCPU. But, that's OK because the purpose of the IPI
233 * is to ensure the vCPU returns to OUTSIDE_GUEST_MODE, which is
234 * satisfied if the vCPU migrates. Entering READING_SHADOW_PAGE_TABLES
235 * after this point is also OK, as the requirement is only that KVM wait
236 * for vCPUs that were reading SPTEs _before_ any changes were
237 * finalized. See kvm_vcpu_kick() for more details on handling requests.
238 */
239 if (kvm_request_needs_ipi(vcpu, req)) {
240 cpu = READ_ONCE(vcpu->cpu);
241 if (cpu != -1 && cpu != current_cpu)
242 __cpumask_set_cpu(cpu, tmp);
243 }
244}
245
246bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
247 unsigned long *vcpu_bitmap)
248{
249 struct kvm_vcpu *vcpu;
250 struct cpumask *cpus;
251 int i, me;
252 bool called;
253
254 me = get_cpu();
255
256 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
257 cpumask_clear(cpus);
258
259 for_each_set_bit(i, vcpu_bitmap, KVM_MAX_VCPUS) {
260 vcpu = kvm_get_vcpu(kvm, i);
261 if (!vcpu)
262 continue;
263 kvm_make_vcpu_request(vcpu, req, cpus, me);
264 }
265
266 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
267 put_cpu();
268
269 return called;
270}
271
272bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
273{
274 struct kvm_vcpu *vcpu;
275 struct cpumask *cpus;
276 unsigned long i;
277 bool called;
278 int me;
279
280 me = get_cpu();
281
282 cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
283 cpumask_clear(cpus);
284
285 kvm_for_each_vcpu(i, vcpu, kvm)
286 kvm_make_vcpu_request(vcpu, req, cpus, me);
287
288 called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
289 put_cpu();
290
291 return called;
292}
293EXPORT_SYMBOL_GPL(kvm_make_all_cpus_request);
294
295void kvm_flush_remote_tlbs(struct kvm *kvm)
296{
297 ++kvm->stat.generic.remote_tlb_flush_requests;
298
299 /*
300 * We want to publish modifications to the page tables before reading
301 * mode. Pairs with a memory barrier in arch-specific code.
302 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
303 * and smp_mb in walk_shadow_page_lockless_begin/end.
304 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
305 *
306 * There is already an smp_mb__after_atomic() before
307 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
308 * barrier here.
309 */
310 if (!kvm_arch_flush_remote_tlbs(kvm)
311 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
312 ++kvm->stat.generic.remote_tlb_flush;
313}
314EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
315
316void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
317{
318 if (!kvm_arch_flush_remote_tlbs_range(kvm, gfn, nr_pages))
319 return;
320
321 /*
322 * Fall back to a flushing entire TLBs if the architecture range-based
323 * TLB invalidation is unsupported or can't be performed for whatever
324 * reason.
325 */
326 kvm_flush_remote_tlbs(kvm);
327}
328
329void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
330 const struct kvm_memory_slot *memslot)
331{
332 /*
333 * All current use cases for flushing the TLBs for a specific memslot
334 * are related to dirty logging, and many do the TLB flush out of
335 * mmu_lock. The interaction between the various operations on memslot
336 * must be serialized by slots_locks to ensure the TLB flush from one
337 * operation is observed by any other operation on the same memslot.
338 */
339 lockdep_assert_held(&kvm->slots_lock);
340 kvm_flush_remote_tlbs_range(kvm, memslot->base_gfn, memslot->npages);
341}
342
343static void kvm_flush_shadow_all(struct kvm *kvm)
344{
345 kvm_arch_flush_shadow_all(kvm);
346 kvm_arch_guest_memory_reclaimed(kvm);
347}
348
349#ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
350static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc,
351 gfp_t gfp_flags)
352{
353 void *page;
354
355 gfp_flags |= mc->gfp_zero;
356
357 if (mc->kmem_cache)
358 return kmem_cache_alloc(mc->kmem_cache, gfp_flags);
359
360 page = (void *)__get_free_page(gfp_flags);
361 if (page && mc->init_value)
362 memset64(page, mc->init_value, PAGE_SIZE / sizeof(u64));
363 return page;
364}
365
366int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min)
367{
368 gfp_t gfp = mc->gfp_custom ? mc->gfp_custom : GFP_KERNEL_ACCOUNT;
369 void *obj;
370
371 if (mc->nobjs >= min)
372 return 0;
373
374 if (unlikely(!mc->objects)) {
375 if (WARN_ON_ONCE(!capacity))
376 return -EIO;
377
378 /*
379 * Custom init values can be used only for page allocations,
380 * and obviously conflict with __GFP_ZERO.
381 */
382 if (WARN_ON_ONCE(mc->init_value && (mc->kmem_cache || mc->gfp_zero)))
383 return -EIO;
384
385 mc->objects = kvmalloc_array(capacity, sizeof(void *), gfp);
386 if (!mc->objects)
387 return -ENOMEM;
388
389 mc->capacity = capacity;
390 }
391
392 /* It is illegal to request a different capacity across topups. */
393 if (WARN_ON_ONCE(mc->capacity != capacity))
394 return -EIO;
395
396 while (mc->nobjs < mc->capacity) {
397 obj = mmu_memory_cache_alloc_obj(mc, gfp);
398 if (!obj)
399 return mc->nobjs >= min ? 0 : -ENOMEM;
400 mc->objects[mc->nobjs++] = obj;
401 }
402 return 0;
403}
404
405int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min)
406{
407 return __kvm_mmu_topup_memory_cache(mc, KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE, min);
408}
409
410int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc)
411{
412 return mc->nobjs;
413}
414
415void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
416{
417 while (mc->nobjs) {
418 if (mc->kmem_cache)
419 kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]);
420 else
421 free_page((unsigned long)mc->objects[--mc->nobjs]);
422 }
423
424 kvfree(mc->objects);
425
426 mc->objects = NULL;
427 mc->capacity = 0;
428}
429
430void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
431{
432 void *p;
433
434 if (WARN_ON(!mc->nobjs))
435 p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT);
436 else
437 p = mc->objects[--mc->nobjs];
438 BUG_ON(!p);
439 return p;
440}
441#endif
442
443static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
444{
445 mutex_init(&vcpu->mutex);
446 vcpu->cpu = -1;
447 vcpu->kvm = kvm;
448 vcpu->vcpu_id = id;
449 vcpu->pid = NULL;
450 rwlock_init(&vcpu->pid_lock);
451#ifndef __KVM_HAVE_ARCH_WQP
452 rcuwait_init(&vcpu->wait);
453#endif
454 kvm_async_pf_vcpu_init(vcpu);
455
456 kvm_vcpu_set_in_spin_loop(vcpu, false);
457 kvm_vcpu_set_dy_eligible(vcpu, false);
458 vcpu->preempted = false;
459 vcpu->ready = false;
460 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
461 vcpu->last_used_slot = NULL;
462
463 /* Fill the stats id string for the vcpu */
464 snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d",
465 task_pid_nr(current), id);
466}
467
468static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
469{
470 kvm_arch_vcpu_destroy(vcpu);
471 kvm_dirty_ring_free(&vcpu->dirty_ring);
472
473 /*
474 * No need for rcu_read_lock as VCPU_RUN is the only place that changes
475 * the vcpu->pid pointer, and at destruction time all file descriptors
476 * are already gone.
477 */
478 put_pid(vcpu->pid);
479
480 free_page((unsigned long)vcpu->run);
481 kmem_cache_free(kvm_vcpu_cache, vcpu);
482}
483
484void kvm_destroy_vcpus(struct kvm *kvm)
485{
486 unsigned long i;
487 struct kvm_vcpu *vcpu;
488
489 kvm_for_each_vcpu(i, vcpu, kvm) {
490 kvm_vcpu_destroy(vcpu);
491 xa_erase(&kvm->vcpu_array, i);
492 }
493
494 atomic_set(&kvm->online_vcpus, 0);
495}
496EXPORT_SYMBOL_GPL(kvm_destroy_vcpus);
497
498#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
499static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
500{
501 return container_of(mn, struct kvm, mmu_notifier);
502}
503
504typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
505
506typedef void (*on_lock_fn_t)(struct kvm *kvm);
507
508struct kvm_mmu_notifier_range {
509 /*
510 * 64-bit addresses, as KVM notifiers can operate on host virtual
511 * addresses (unsigned long) and guest physical addresses (64-bit).
512 */
513 u64 start;
514 u64 end;
515 union kvm_mmu_notifier_arg arg;
516 gfn_handler_t handler;
517 on_lock_fn_t on_lock;
518 bool flush_on_ret;
519 bool may_block;
520};
521
522/*
523 * The inner-most helper returns a tuple containing the return value from the
524 * arch- and action-specific handler, plus a flag indicating whether or not at
525 * least one memslot was found, i.e. if the handler found guest memory.
526 *
527 * Note, most notifiers are averse to booleans, so even though KVM tracks the
528 * return from arch code as a bool, outer helpers will cast it to an int. :-(
529 */
530typedef struct kvm_mmu_notifier_return {
531 bool ret;
532 bool found_memslot;
533} kvm_mn_ret_t;
534
535/*
536 * Use a dedicated stub instead of NULL to indicate that there is no callback
537 * function/handler. The compiler technically can't guarantee that a real
538 * function will have a non-zero address, and so it will generate code to
539 * check for !NULL, whereas comparing against a stub will be elided at compile
540 * time (unless the compiler is getting long in the tooth, e.g. gcc 4.9).
541 */
542static void kvm_null_fn(void)
543{
544
545}
546#define IS_KVM_NULL_FN(fn) ((fn) == (void *)kvm_null_fn)
547
548/* Iterate over each memslot intersecting [start, last] (inclusive) range */
549#define kvm_for_each_memslot_in_hva_range(node, slots, start, last) \
550 for (node = interval_tree_iter_first(&slots->hva_tree, start, last); \
551 node; \
552 node = interval_tree_iter_next(node, start, last)) \
553
554static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
555 const struct kvm_mmu_notifier_range *range)
556{
557 struct kvm_mmu_notifier_return r = {
558 .ret = false,
559 .found_memslot = false,
560 };
561 struct kvm_gfn_range gfn_range;
562 struct kvm_memory_slot *slot;
563 struct kvm_memslots *slots;
564 int i, idx;
565
566 if (WARN_ON_ONCE(range->end <= range->start))
567 return r;
568
569 /* A null handler is allowed if and only if on_lock() is provided. */
570 if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
571 IS_KVM_NULL_FN(range->handler)))
572 return r;
573
574 idx = srcu_read_lock(&kvm->srcu);
575
576 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
577 struct interval_tree_node *node;
578
579 slots = __kvm_memslots(kvm, i);
580 kvm_for_each_memslot_in_hva_range(node, slots,
581 range->start, range->end - 1) {
582 unsigned long hva_start, hva_end;
583
584 slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
585 hva_start = max_t(unsigned long, range->start, slot->userspace_addr);
586 hva_end = min_t(unsigned long, range->end,
587 slot->userspace_addr + (slot->npages << PAGE_SHIFT));
588
589 /*
590 * To optimize for the likely case where the address
591 * range is covered by zero or one memslots, don't
592 * bother making these conditional (to avoid writes on
593 * the second or later invocation of the handler).
594 */
595 gfn_range.arg = range->arg;
596 gfn_range.may_block = range->may_block;
597
598 /*
599 * {gfn(page) | page intersects with [hva_start, hva_end)} =
600 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
601 */
602 gfn_range.start = hva_to_gfn_memslot(hva_start, slot);
603 gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
604 gfn_range.slot = slot;
605
606 if (!r.found_memslot) {
607 r.found_memslot = true;
608 KVM_MMU_LOCK(kvm);
609 if (!IS_KVM_NULL_FN(range->on_lock))
610 range->on_lock(kvm);
611
612 if (IS_KVM_NULL_FN(range->handler))
613 goto mmu_unlock;
614 }
615 r.ret |= range->handler(kvm, &gfn_range);
616 }
617 }
618
619 if (range->flush_on_ret && r.ret)
620 kvm_flush_remote_tlbs(kvm);
621
622mmu_unlock:
623 if (r.found_memslot)
624 KVM_MMU_UNLOCK(kvm);
625
626 srcu_read_unlock(&kvm->srcu, idx);
627
628 return r;
629}
630
631static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
632 unsigned long start,
633 unsigned long end,
634 gfn_handler_t handler,
635 bool flush_on_ret)
636{
637 struct kvm *kvm = mmu_notifier_to_kvm(mn);
638 const struct kvm_mmu_notifier_range range = {
639 .start = start,
640 .end = end,
641 .handler = handler,
642 .on_lock = (void *)kvm_null_fn,
643 .flush_on_ret = flush_on_ret,
644 .may_block = false,
645 };
646
647 return __kvm_handle_hva_range(kvm, &range).ret;
648}
649
650static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
651 unsigned long start,
652 unsigned long end,
653 gfn_handler_t handler)
654{
655 return kvm_handle_hva_range(mn, start, end, handler, false);
656}
657
658void kvm_mmu_invalidate_begin(struct kvm *kvm)
659{
660 lockdep_assert_held_write(&kvm->mmu_lock);
661 /*
662 * The count increase must become visible at unlock time as no
663 * spte can be established without taking the mmu_lock and
664 * count is also read inside the mmu_lock critical section.
665 */
666 kvm->mmu_invalidate_in_progress++;
667
668 if (likely(kvm->mmu_invalidate_in_progress == 1)) {
669 kvm->mmu_invalidate_range_start = INVALID_GPA;
670 kvm->mmu_invalidate_range_end = INVALID_GPA;
671 }
672}
673
674void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
675{
676 lockdep_assert_held_write(&kvm->mmu_lock);
677
678 WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);
679
680 if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
681 kvm->mmu_invalidate_range_start = start;
682 kvm->mmu_invalidate_range_end = end;
683 } else {
684 /*
685 * Fully tracking multiple concurrent ranges has diminishing
686 * returns. Keep things simple and just find the minimal range
687 * which includes the current and new ranges. As there won't be
688 * enough information to subtract a range after its invalidate
689 * completes, any ranges invalidated concurrently will
690 * accumulate and persist until all outstanding invalidates
691 * complete.
692 */
693 kvm->mmu_invalidate_range_start =
694 min(kvm->mmu_invalidate_range_start, start);
695 kvm->mmu_invalidate_range_end =
696 max(kvm->mmu_invalidate_range_end, end);
697 }
698}
699
700bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
701{
702 kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
703 return kvm_unmap_gfn_range(kvm, range);
704}
705
706static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
707 const struct mmu_notifier_range *range)
708{
709 struct kvm *kvm = mmu_notifier_to_kvm(mn);
710 const struct kvm_mmu_notifier_range hva_range = {
711 .start = range->start,
712 .end = range->end,
713 .handler = kvm_mmu_unmap_gfn_range,
714 .on_lock = kvm_mmu_invalidate_begin,
715 .flush_on_ret = true,
716 .may_block = mmu_notifier_range_blockable(range),
717 };
718
719 trace_kvm_unmap_hva_range(range->start, range->end);
720
721 /*
722 * Prevent memslot modification between range_start() and range_end()
723 * so that conditionally locking provides the same result in both
724 * functions. Without that guarantee, the mmu_invalidate_in_progress
725 * adjustments will be imbalanced.
726 *
727 * Pairs with the decrement in range_end().
728 */
729 spin_lock(&kvm->mn_invalidate_lock);
730 kvm->mn_active_invalidate_count++;
731 spin_unlock(&kvm->mn_invalidate_lock);
732
733 /*
734 * Invalidate pfn caches _before_ invalidating the secondary MMUs, i.e.
735 * before acquiring mmu_lock, to avoid holding mmu_lock while acquiring
736 * each cache's lock. There are relatively few caches in existence at
737 * any given time, and the caches themselves can check for hva overlap,
738 * i.e. don't need to rely on memslot overlap checks for performance.
739 * Because this runs without holding mmu_lock, the pfn caches must use
740 * mn_active_invalidate_count (see above) instead of
741 * mmu_invalidate_in_progress.
742 */
743 gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end);
744
745 /*
746 * If one or more memslots were found and thus zapped, notify arch code
747 * that guest memory has been reclaimed. This needs to be done *after*
748 * dropping mmu_lock, as x86's reclaim path is slooooow.
749 */
750 if (__kvm_handle_hva_range(kvm, &hva_range).found_memslot)
751 kvm_arch_guest_memory_reclaimed(kvm);
752
753 return 0;
754}
755
756void kvm_mmu_invalidate_end(struct kvm *kvm)
757{
758 lockdep_assert_held_write(&kvm->mmu_lock);
759
760 /*
761 * This sequence increase will notify the kvm page fault that
762 * the page that is going to be mapped in the spte could have
763 * been freed.
764 */
765 kvm->mmu_invalidate_seq++;
766 smp_wmb();
767 /*
768 * The above sequence increase must be visible before the
769 * below count decrease, which is ensured by the smp_wmb above
770 * in conjunction with the smp_rmb in mmu_invalidate_retry().
771 */
772 kvm->mmu_invalidate_in_progress--;
773 KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
774
775 /*
776 * Assert that at least one range was added between start() and end().
777 * Not adding a range isn't fatal, but it is a KVM bug.
778 */
779 WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
780}
781
782static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
783 const struct mmu_notifier_range *range)
784{
785 struct kvm *kvm = mmu_notifier_to_kvm(mn);
786 const struct kvm_mmu_notifier_range hva_range = {
787 .start = range->start,
788 .end = range->end,
789 .handler = (void *)kvm_null_fn,
790 .on_lock = kvm_mmu_invalidate_end,
791 .flush_on_ret = false,
792 .may_block = mmu_notifier_range_blockable(range),
793 };
794 bool wake;
795
796 __kvm_handle_hva_range(kvm, &hva_range);
797
798 /* Pairs with the increment in range_start(). */
799 spin_lock(&kvm->mn_invalidate_lock);
800 if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count))
801 --kvm->mn_active_invalidate_count;
802 wake = !kvm->mn_active_invalidate_count;
803 spin_unlock(&kvm->mn_invalidate_lock);
804
805 /*
806 * There can only be one waiter, since the wait happens under
807 * slots_lock.
808 */
809 if (wake)
810 rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
811}
812
813static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
814 struct mm_struct *mm,
815 unsigned long start,
816 unsigned long end)
817{
818 trace_kvm_age_hva(start, end);
819
820 return kvm_handle_hva_range(mn, start, end, kvm_age_gfn,
821 !IS_ENABLED(CONFIG_KVM_ELIDE_TLB_FLUSH_IF_YOUNG));
822}
823
824static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
825 struct mm_struct *mm,
826 unsigned long start,
827 unsigned long end)
828{
829 trace_kvm_age_hva(start, end);
830
831 /*
832 * Even though we do not flush TLB, this will still adversely
833 * affect performance on pre-Haswell Intel EPT, where there is
834 * no EPT Access Bit to clear so that we have to tear down EPT
835 * tables instead. If we find this unacceptable, we can always
836 * add a parameter to kvm_age_hva so that it effectively doesn't
837 * do anything on clear_young.
838 *
839 * Also note that currently we never issue secondary TLB flushes
840 * from clear_young, leaving this job up to the regular system
841 * cadence. If we find this inaccurate, we might come up with a
842 * more sophisticated heuristic later.
843 */
844 return kvm_handle_hva_range_no_flush(mn, start, end, kvm_age_gfn);
845}
846
847static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
848 struct mm_struct *mm,
849 unsigned long address)
850{
851 trace_kvm_test_age_hva(address);
852
853 return kvm_handle_hva_range_no_flush(mn, address, address + 1,
854 kvm_test_age_gfn);
855}
856
857static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
858 struct mm_struct *mm)
859{
860 struct kvm *kvm = mmu_notifier_to_kvm(mn);
861 int idx;
862
863 idx = srcu_read_lock(&kvm->srcu);
864 kvm_flush_shadow_all(kvm);
865 srcu_read_unlock(&kvm->srcu, idx);
866}
867
868static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
869 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
870 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
871 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
872 .clear_young = kvm_mmu_notifier_clear_young,
873 .test_young = kvm_mmu_notifier_test_young,
874 .release = kvm_mmu_notifier_release,
875};
876
877static int kvm_init_mmu_notifier(struct kvm *kvm)
878{
879 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
880 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
881}
882
883#else /* !CONFIG_KVM_GENERIC_MMU_NOTIFIER */
884
885static int kvm_init_mmu_notifier(struct kvm *kvm)
886{
887 return 0;
888}
889
890#endif /* CONFIG_KVM_GENERIC_MMU_NOTIFIER */
891
892#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
893static int kvm_pm_notifier_call(struct notifier_block *bl,
894 unsigned long state,
895 void *unused)
896{
897 struct kvm *kvm = container_of(bl, struct kvm, pm_notifier);
898
899 return kvm_arch_pm_notifier(kvm, state);
900}
901
902static void kvm_init_pm_notifier(struct kvm *kvm)
903{
904 kvm->pm_notifier.notifier_call = kvm_pm_notifier_call;
905 /* Suspend KVM before we suspend ftrace, RCU, etc. */
906 kvm->pm_notifier.priority = INT_MAX;
907 register_pm_notifier(&kvm->pm_notifier);
908}
909
910static void kvm_destroy_pm_notifier(struct kvm *kvm)
911{
912 unregister_pm_notifier(&kvm->pm_notifier);
913}
914#else /* !CONFIG_HAVE_KVM_PM_NOTIFIER */
915static void kvm_init_pm_notifier(struct kvm *kvm)
916{
917}
918
919static void kvm_destroy_pm_notifier(struct kvm *kvm)
920{
921}
922#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
923
924static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
925{
926 if (!memslot->dirty_bitmap)
927 return;
928
929 vfree(memslot->dirty_bitmap);
930 memslot->dirty_bitmap = NULL;
931}
932
933/* This does not remove the slot from struct kvm_memslots data structures */
934static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
935{
936 if (slot->flags & KVM_MEM_GUEST_MEMFD)
937 kvm_gmem_unbind(slot);
938
939 kvm_destroy_dirty_bitmap(slot);
940
941 kvm_arch_free_memslot(kvm, slot);
942
943 kfree(slot);
944}
945
946static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
947{
948 struct hlist_node *idnode;
949 struct kvm_memory_slot *memslot;
950 int bkt;
951
952 /*
953 * The same memslot objects live in both active and inactive sets,
954 * arbitrarily free using index '1' so the second invocation of this
955 * function isn't operating over a structure with dangling pointers
956 * (even though this function isn't actually touching them).
957 */
958 if (!slots->node_idx)
959 return;
960
961 hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1])
962 kvm_free_memslot(kvm, memslot);
963}
964
965static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc)
966{
967 switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
968 case KVM_STATS_TYPE_INSTANT:
969 return 0444;
970 case KVM_STATS_TYPE_CUMULATIVE:
971 case KVM_STATS_TYPE_PEAK:
972 default:
973 return 0644;
974 }
975}
976
977
978static void kvm_destroy_vm_debugfs(struct kvm *kvm)
979{
980 int i;
981 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
982 kvm_vcpu_stats_header.num_desc;
983
984 if (IS_ERR(kvm->debugfs_dentry))
985 return;
986
987 debugfs_remove_recursive(kvm->debugfs_dentry);
988
989 if (kvm->debugfs_stat_data) {
990 for (i = 0; i < kvm_debugfs_num_entries; i++)
991 kfree(kvm->debugfs_stat_data[i]);
992 kfree(kvm->debugfs_stat_data);
993 }
994}
995
996static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname)
997{
998 static DEFINE_MUTEX(kvm_debugfs_lock);
999 struct dentry *dent;
1000 char dir_name[ITOA_MAX_LEN * 2];
1001 struct kvm_stat_data *stat_data;
1002 const struct _kvm_stats_desc *pdesc;
1003 int i, ret = -ENOMEM;
1004 int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
1005 kvm_vcpu_stats_header.num_desc;
1006
1007 if (!debugfs_initialized())
1008 return 0;
1009
1010 snprintf(dir_name, sizeof(dir_name), "%d-%s", task_pid_nr(current), fdname);
1011 mutex_lock(&kvm_debugfs_lock);
1012 dent = debugfs_lookup(dir_name, kvm_debugfs_dir);
1013 if (dent) {
1014 pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name);
1015 dput(dent);
1016 mutex_unlock(&kvm_debugfs_lock);
1017 return 0;
1018 }
1019 dent = debugfs_create_dir(dir_name, kvm_debugfs_dir);
1020 mutex_unlock(&kvm_debugfs_lock);
1021 if (IS_ERR(dent))
1022 return 0;
1023
1024 kvm->debugfs_dentry = dent;
1025 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
1026 sizeof(*kvm->debugfs_stat_data),
1027 GFP_KERNEL_ACCOUNT);
1028 if (!kvm->debugfs_stat_data)
1029 goto out_err;
1030
1031 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
1032 pdesc = &kvm_vm_stats_desc[i];
1033 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
1034 if (!stat_data)
1035 goto out_err;
1036
1037 stat_data->kvm = kvm;
1038 stat_data->desc = pdesc;
1039 stat_data->kind = KVM_STAT_VM;
1040 kvm->debugfs_stat_data[i] = stat_data;
1041 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
1042 kvm->debugfs_dentry, stat_data,
1043 &stat_fops_per_vm);
1044 }
1045
1046 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
1047 pdesc = &kvm_vcpu_stats_desc[i];
1048 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
1049 if (!stat_data)
1050 goto out_err;
1051
1052 stat_data->kvm = kvm;
1053 stat_data->desc = pdesc;
1054 stat_data->kind = KVM_STAT_VCPU;
1055 kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;
1056 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
1057 kvm->debugfs_dentry, stat_data,
1058 &stat_fops_per_vm);
1059 }
1060
1061 kvm_arch_create_vm_debugfs(kvm);
1062 return 0;
1063out_err:
1064 kvm_destroy_vm_debugfs(kvm);
1065 return ret;
1066}
1067
1068/*
1069 * Called after the VM is otherwise initialized, but just before adding it to
1070 * the vm_list.
1071 */
1072int __weak kvm_arch_post_init_vm(struct kvm *kvm)
1073{
1074 return 0;
1075}
1076
1077/*
1078 * Called just after removing the VM from the vm_list, but before doing any
1079 * other destruction.
1080 */
1081void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
1082{
1083}
1084
1085/*
1086 * Called after per-vm debugfs created. When called kvm->debugfs_dentry should
1087 * be setup already, so we can create arch-specific debugfs entries under it.
1088 * Cleanup should be automatic done in kvm_destroy_vm_debugfs() recursively, so
1089 * a per-arch destroy interface is not needed.
1090 */
1091void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
1092{
1093}
1094
1095static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
1096{
1097 struct kvm *kvm = kvm_arch_alloc_vm();
1098 struct kvm_memslots *slots;
1099 int r, i, j;
1100
1101 if (!kvm)
1102 return ERR_PTR(-ENOMEM);
1103
1104 KVM_MMU_LOCK_INIT(kvm);
1105 mmgrab(current->mm);
1106 kvm->mm = current->mm;
1107 kvm_eventfd_init(kvm);
1108 mutex_init(&kvm->lock);
1109 mutex_init(&kvm->irq_lock);
1110 mutex_init(&kvm->slots_lock);
1111 mutex_init(&kvm->slots_arch_lock);
1112 spin_lock_init(&kvm->mn_invalidate_lock);
1113 rcuwait_init(&kvm->mn_memslots_update_rcuwait);
1114 xa_init(&kvm->vcpu_array);
1115#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1116 xa_init(&kvm->mem_attr_array);
1117#endif
1118
1119 INIT_LIST_HEAD(&kvm->gpc_list);
1120 spin_lock_init(&kvm->gpc_lock);
1121
1122 INIT_LIST_HEAD(&kvm->devices);
1123 kvm->max_vcpus = KVM_MAX_VCPUS;
1124
1125 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
1126
1127 /*
1128 * Force subsequent debugfs file creations to fail if the VM directory
1129 * is not created (by kvm_create_vm_debugfs()).
1130 */
1131 kvm->debugfs_dentry = ERR_PTR(-ENOENT);
1132
1133 snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",
1134 task_pid_nr(current));
1135
1136 r = -ENOMEM;
1137 if (init_srcu_struct(&kvm->srcu))
1138 goto out_err_no_srcu;
1139 if (init_srcu_struct(&kvm->irq_srcu))
1140 goto out_err_no_irq_srcu;
1141
1142 r = kvm_init_irq_routing(kvm);
1143 if (r)
1144 goto out_err_no_irq_routing;
1145
1146 refcount_set(&kvm->users_count, 1);
1147
1148 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
1149 for (j = 0; j < 2; j++) {
1150 slots = &kvm->__memslots[i][j];
1151
1152 atomic_long_set(&slots->last_used_slot, (unsigned long)NULL);
1153 slots->hva_tree = RB_ROOT_CACHED;
1154 slots->gfn_tree = RB_ROOT;
1155 hash_init(slots->id_hash);
1156 slots->node_idx = j;
1157
1158 /* Generations must be different for each address space. */
1159 slots->generation = i;
1160 }
1161
1162 rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
1163 }
1164
1165 r = -ENOMEM;
1166 for (i = 0; i < KVM_NR_BUSES; i++) {
1167 rcu_assign_pointer(kvm->buses[i],
1168 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
1169 if (!kvm->buses[i])
1170 goto out_err_no_arch_destroy_vm;
1171 }
1172
1173 r = kvm_arch_init_vm(kvm, type);
1174 if (r)
1175 goto out_err_no_arch_destroy_vm;
1176
1177 r = kvm_enable_virtualization();
1178 if (r)
1179 goto out_err_no_disable;
1180
1181#ifdef CONFIG_HAVE_KVM_IRQCHIP
1182 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
1183#endif
1184
1185 r = kvm_init_mmu_notifier(kvm);
1186 if (r)
1187 goto out_err_no_mmu_notifier;
1188
1189 r = kvm_coalesced_mmio_init(kvm);
1190 if (r < 0)
1191 goto out_no_coalesced_mmio;
1192
1193 r = kvm_create_vm_debugfs(kvm, fdname);
1194 if (r)
1195 goto out_err_no_debugfs;
1196
1197 r = kvm_arch_post_init_vm(kvm);
1198 if (r)
1199 goto out_err;
1200
1201 mutex_lock(&kvm_lock);
1202 list_add(&kvm->vm_list, &vm_list);
1203 mutex_unlock(&kvm_lock);
1204
1205 preempt_notifier_inc();
1206 kvm_init_pm_notifier(kvm);
1207
1208 return kvm;
1209
1210out_err:
1211 kvm_destroy_vm_debugfs(kvm);
1212out_err_no_debugfs:
1213 kvm_coalesced_mmio_free(kvm);
1214out_no_coalesced_mmio:
1215#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
1216 if (kvm->mmu_notifier.ops)
1217 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
1218#endif
1219out_err_no_mmu_notifier:
1220 kvm_disable_virtualization();
1221out_err_no_disable:
1222 kvm_arch_destroy_vm(kvm);
1223out_err_no_arch_destroy_vm:
1224 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
1225 for (i = 0; i < KVM_NR_BUSES; i++)
1226 kfree(kvm_get_bus(kvm, i));
1227 kvm_free_irq_routing(kvm);
1228out_err_no_irq_routing:
1229 cleanup_srcu_struct(&kvm->irq_srcu);
1230out_err_no_irq_srcu:
1231 cleanup_srcu_struct(&kvm->srcu);
1232out_err_no_srcu:
1233 kvm_arch_free_vm(kvm);
1234 mmdrop(current->mm);
1235 return ERR_PTR(r);
1236}
1237
1238static void kvm_destroy_devices(struct kvm *kvm)
1239{
1240 struct kvm_device *dev, *tmp;
1241
1242 /*
1243 * We do not need to take the kvm->lock here, because nobody else
1244 * has a reference to the struct kvm at this point and therefore
1245 * cannot access the devices list anyhow.
1246 *
1247 * The device list is generally managed as an rculist, but list_del()
1248 * is used intentionally here. If a bug in KVM introduced a reader that
1249 * was not backed by a reference on the kvm struct, the hope is that
1250 * it'd consume the poisoned forward pointer instead of suffering a
1251 * use-after-free, even though this cannot be guaranteed.
1252 */
1253 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
1254 list_del(&dev->vm_node);
1255 dev->ops->destroy(dev);
1256 }
1257}
1258
1259static void kvm_destroy_vm(struct kvm *kvm)
1260{
1261 int i;
1262 struct mm_struct *mm = kvm->mm;
1263
1264 kvm_destroy_pm_notifier(kvm);
1265 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
1266 kvm_destroy_vm_debugfs(kvm);
1267 kvm_arch_sync_events(kvm);
1268 mutex_lock(&kvm_lock);
1269 list_del(&kvm->vm_list);
1270 mutex_unlock(&kvm_lock);
1271 kvm_arch_pre_destroy_vm(kvm);
1272
1273 kvm_free_irq_routing(kvm);
1274 for (i = 0; i < KVM_NR_BUSES; i++) {
1275 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
1276
1277 if (bus)
1278 kvm_io_bus_destroy(bus);
1279 kvm->buses[i] = NULL;
1280 }
1281 kvm_coalesced_mmio_free(kvm);
1282#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
1283 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
1284 /*
1285 * At this point, pending calls to invalidate_range_start()
1286 * have completed but no more MMU notifiers will run, so
1287 * mn_active_invalidate_count may remain unbalanced.
1288 * No threads can be waiting in kvm_swap_active_memslots() as the
1289 * last reference on KVM has been dropped, but freeing
1290 * memslots would deadlock without this manual intervention.
1291 *
1292 * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU
1293 * notifier between a start() and end(), then there shouldn't be any
1294 * in-progress invalidations.
1295 */
1296 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
1297 if (kvm->mn_active_invalidate_count)
1298 kvm->mn_active_invalidate_count = 0;
1299 else
1300 WARN_ON(kvm->mmu_invalidate_in_progress);
1301#else
1302 kvm_flush_shadow_all(kvm);
1303#endif
1304 kvm_arch_destroy_vm(kvm);
1305 kvm_destroy_devices(kvm);
1306 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
1307 kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
1308 kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
1309 }
1310 cleanup_srcu_struct(&kvm->irq_srcu);
1311 cleanup_srcu_struct(&kvm->srcu);
1312#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
1313 xa_destroy(&kvm->mem_attr_array);
1314#endif
1315 kvm_arch_free_vm(kvm);
1316 preempt_notifier_dec();
1317 kvm_disable_virtualization();
1318 mmdrop(mm);
1319}
1320
1321void kvm_get_kvm(struct kvm *kvm)
1322{
1323 refcount_inc(&kvm->users_count);
1324}
1325EXPORT_SYMBOL_GPL(kvm_get_kvm);
1326
1327/*
1328 * Make sure the vm is not during destruction, which is a safe version of
1329 * kvm_get_kvm(). Return true if kvm referenced successfully, false otherwise.
1330 */
1331bool kvm_get_kvm_safe(struct kvm *kvm)
1332{
1333 return refcount_inc_not_zero(&kvm->users_count);
1334}
1335EXPORT_SYMBOL_GPL(kvm_get_kvm_safe);
1336
1337void kvm_put_kvm(struct kvm *kvm)
1338{
1339 if (refcount_dec_and_test(&kvm->users_count))
1340 kvm_destroy_vm(kvm);
1341}
1342EXPORT_SYMBOL_GPL(kvm_put_kvm);
1343
1344/*
1345 * Used to put a reference that was taken on behalf of an object associated
1346 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
1347 * of the new file descriptor fails and the reference cannot be transferred to
1348 * its final owner. In such cases, the caller is still actively using @kvm and
1349 * will fail miserably if the refcount unexpectedly hits zero.
1350 */
1351void kvm_put_kvm_no_destroy(struct kvm *kvm)
1352{
1353 WARN_ON(refcount_dec_and_test(&kvm->users_count));
1354}
1355EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
1356
1357static int kvm_vm_release(struct inode *inode, struct file *filp)
1358{
1359 struct kvm *kvm = filp->private_data;
1360
1361 kvm_irqfd_release(kvm);
1362
1363 kvm_put_kvm(kvm);
1364 return 0;
1365}
1366
1367/*
1368 * Allocation size is twice as large as the actual dirty bitmap size.
1369 * See kvm_vm_ioctl_get_dirty_log() why this is needed.
1370 */
1371static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)
1372{
1373 unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot);
1374
1375 memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT);
1376 if (!memslot->dirty_bitmap)
1377 return -ENOMEM;
1378
1379 return 0;
1380}
1381
1382static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)
1383{
1384 struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
1385 int node_idx_inactive = active->node_idx ^ 1;
1386
1387 return &kvm->__memslots[as_id][node_idx_inactive];
1388}
1389
1390/*
1391 * Helper to get the address space ID when one of memslot pointers may be NULL.
1392 * This also serves as a sanity that at least one of the pointers is non-NULL,
1393 * and that their address space IDs don't diverge.
1394 */
1395static int kvm_memslots_get_as_id(struct kvm_memory_slot *a,
1396 struct kvm_memory_slot *b)
1397{
1398 if (WARN_ON_ONCE(!a && !b))
1399 return 0;
1400
1401 if (!a)
1402 return b->as_id;
1403 if (!b)
1404 return a->as_id;
1405
1406 WARN_ON_ONCE(a->as_id != b->as_id);
1407 return a->as_id;
1408}
1409
1410static void kvm_insert_gfn_node(struct kvm_memslots *slots,
1411 struct kvm_memory_slot *slot)
1412{
1413 struct rb_root *gfn_tree = &slots->gfn_tree;
1414 struct rb_node **node, *parent;
1415 int idx = slots->node_idx;
1416
1417 parent = NULL;
1418 for (node = &gfn_tree->rb_node; *node; ) {
1419 struct kvm_memory_slot *tmp;
1420
1421 tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
1422 parent = *node;
1423 if (slot->base_gfn < tmp->base_gfn)
1424 node = &(*node)->rb_left;
1425 else if (slot->base_gfn > tmp->base_gfn)
1426 node = &(*node)->rb_right;
1427 else
1428 BUG();
1429 }
1430
1431 rb_link_node(&slot->gfn_node[idx], parent, node);
1432 rb_insert_color(&slot->gfn_node[idx], gfn_tree);
1433}
1434
1435static void kvm_erase_gfn_node(struct kvm_memslots *slots,
1436 struct kvm_memory_slot *slot)
1437{
1438 rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree);
1439}
1440
1441static void kvm_replace_gfn_node(struct kvm_memslots *slots,
1442 struct kvm_memory_slot *old,
1443 struct kvm_memory_slot *new)
1444{
1445 int idx = slots->node_idx;
1446
1447 WARN_ON_ONCE(old->base_gfn != new->base_gfn);
1448
1449 rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx],
1450 &slots->gfn_tree);
1451}
1452
1453/*
1454 * Replace @old with @new in the inactive memslots.
1455 *
1456 * With NULL @old this simply adds @new.
1457 * With NULL @new this simply removes @old.
1458 *
1459 * If @new is non-NULL its hva_node[slots_idx] range has to be set
1460 * appropriately.
1461 */
1462static void kvm_replace_memslot(struct kvm *kvm,
1463 struct kvm_memory_slot *old,
1464 struct kvm_memory_slot *new)
1465{
1466 int as_id = kvm_memslots_get_as_id(old, new);
1467 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1468 int idx = slots->node_idx;
1469
1470 if (old) {
1471 hash_del(&old->id_node[idx]);
1472 interval_tree_remove(&old->hva_node[idx], &slots->hva_tree);
1473
1474 if ((long)old == atomic_long_read(&slots->last_used_slot))
1475 atomic_long_set(&slots->last_used_slot, (long)new);
1476
1477 if (!new) {
1478 kvm_erase_gfn_node(slots, old);
1479 return;
1480 }
1481 }
1482
1483 /*
1484 * Initialize @new's hva range. Do this even when replacing an @old
1485 * slot, kvm_copy_memslot() deliberately does not touch node data.
1486 */
1487 new->hva_node[idx].start = new->userspace_addr;
1488 new->hva_node[idx].last = new->userspace_addr +
1489 (new->npages << PAGE_SHIFT) - 1;
1490
1491 /*
1492 * (Re)Add the new memslot. There is no O(1) interval_tree_replace(),
1493 * hva_node needs to be swapped with remove+insert even though hva can't
1494 * change when replacing an existing slot.
1495 */
1496 hash_add(slots->id_hash, &new->id_node[idx], new->id);
1497 interval_tree_insert(&new->hva_node[idx], &slots->hva_tree);
1498
1499 /*
1500 * If the memslot gfn is unchanged, rb_replace_node() can be used to
1501 * switch the node in the gfn tree instead of removing the old and
1502 * inserting the new as two separate operations. Replacement is a
1503 * single O(1) operation versus two O(log(n)) operations for
1504 * remove+insert.
1505 */
1506 if (old && old->base_gfn == new->base_gfn) {
1507 kvm_replace_gfn_node(slots, old, new);
1508 } else {
1509 if (old)
1510 kvm_erase_gfn_node(slots, old);
1511 kvm_insert_gfn_node(slots, new);
1512 }
1513}
1514
1515/*
1516 * Flags that do not access any of the extra space of struct
1517 * kvm_userspace_memory_region2. KVM_SET_USER_MEMORY_REGION_V1_FLAGS
1518 * only allows these.
1519 */
1520#define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \
1521 (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)
1522
1523static int check_memory_region_flags(struct kvm *kvm,
1524 const struct kvm_userspace_memory_region2 *mem)
1525{
1526 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
1527
1528 if (kvm_arch_has_private_mem(kvm))
1529 valid_flags |= KVM_MEM_GUEST_MEMFD;
1530
1531 /* Dirty logging private memory is not currently supported. */
1532 if (mem->flags & KVM_MEM_GUEST_MEMFD)
1533 valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
1534
1535 /*
1536 * GUEST_MEMFD is incompatible with read-only memslots, as writes to
1537 * read-only memslots have emulated MMIO, not page fault, semantics,
1538 * and KVM doesn't allow emulated MMIO for private memory.
1539 */
1540 if (kvm_arch_has_readonly_mem(kvm) &&
1541 !(mem->flags & KVM_MEM_GUEST_MEMFD))
1542 valid_flags |= KVM_MEM_READONLY;
1543
1544 if (mem->flags & ~valid_flags)
1545 return -EINVAL;
1546
1547 return 0;
1548}
1549
1550static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
1551{
1552 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1553
1554 /* Grab the generation from the activate memslots. */
1555 u64 gen = __kvm_memslots(kvm, as_id)->generation;
1556
1557 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
1558 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1559
1560 /*
1561 * Do not store the new memslots while there are invalidations in
1562 * progress, otherwise the locking in invalidate_range_start and
1563 * invalidate_range_end will be unbalanced.
1564 */
1565 spin_lock(&kvm->mn_invalidate_lock);
1566 prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);
1567 while (kvm->mn_active_invalidate_count) {
1568 set_current_state(TASK_UNINTERRUPTIBLE);
1569 spin_unlock(&kvm->mn_invalidate_lock);
1570 schedule();
1571 spin_lock(&kvm->mn_invalidate_lock);
1572 }
1573 finish_rcuwait(&kvm->mn_memslots_update_rcuwait);
1574 rcu_assign_pointer(kvm->memslots[as_id], slots);
1575 spin_unlock(&kvm->mn_invalidate_lock);
1576
1577 /*
1578 * Acquired in kvm_set_memslot. Must be released before synchronize
1579 * SRCU below in order to avoid deadlock with another thread
1580 * acquiring the slots_arch_lock in an srcu critical section.
1581 */
1582 mutex_unlock(&kvm->slots_arch_lock);
1583
1584 synchronize_srcu_expedited(&kvm->srcu);
1585
1586 /*
1587 * Increment the new memslot generation a second time, dropping the
1588 * update in-progress flag and incrementing the generation based on
1589 * the number of address spaces. This provides a unique and easily
1590 * identifiable generation number while the memslots are in flux.
1591 */
1592 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1593
1594 /*
1595 * Generations must be unique even across address spaces. We do not need
1596 * a global counter for that, instead the generation space is evenly split
1597 * across address spaces. For example, with two address spaces, address
1598 * space 0 will use generations 0, 2, 4, ... while address space 1 will
1599 * use generations 1, 3, 5, ...
1600 */
1601 gen += kvm_arch_nr_memslot_as_ids(kvm);
1602
1603 kvm_arch_memslots_updated(kvm, gen);
1604
1605 slots->generation = gen;
1606}
1607
1608static int kvm_prepare_memory_region(struct kvm *kvm,
1609 const struct kvm_memory_slot *old,
1610 struct kvm_memory_slot *new,
1611 enum kvm_mr_change change)
1612{
1613 int r;
1614
1615 /*
1616 * If dirty logging is disabled, nullify the bitmap; the old bitmap
1617 * will be freed on "commit". If logging is enabled in both old and
1618 * new, reuse the existing bitmap. If logging is enabled only in the
1619 * new and KVM isn't using a ring buffer, allocate and initialize a
1620 * new bitmap.
1621 */
1622 if (change != KVM_MR_DELETE) {
1623 if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
1624 new->dirty_bitmap = NULL;
1625 else if (old && old->dirty_bitmap)
1626 new->dirty_bitmap = old->dirty_bitmap;
1627 else if (kvm_use_dirty_bitmap(kvm)) {
1628 r = kvm_alloc_dirty_bitmap(new);
1629 if (r)
1630 return r;
1631
1632 if (kvm_dirty_log_manual_protect_and_init_set(kvm))
1633 bitmap_set(new->dirty_bitmap, 0, new->npages);
1634 }
1635 }
1636
1637 r = kvm_arch_prepare_memory_region(kvm, old, new, change);
1638
1639 /* Free the bitmap on failure if it was allocated above. */
1640 if (r && new && new->dirty_bitmap && (!old || !old->dirty_bitmap))
1641 kvm_destroy_dirty_bitmap(new);
1642
1643 return r;
1644}
1645
1646static void kvm_commit_memory_region(struct kvm *kvm,
1647 struct kvm_memory_slot *old,
1648 const struct kvm_memory_slot *new,
1649 enum kvm_mr_change change)
1650{
1651 int old_flags = old ? old->flags : 0;
1652 int new_flags = new ? new->flags : 0;
1653 /*
1654 * Update the total number of memslot pages before calling the arch
1655 * hook so that architectures can consume the result directly.
1656 */
1657 if (change == KVM_MR_DELETE)
1658 kvm->nr_memslot_pages -= old->npages;
1659 else if (change == KVM_MR_CREATE)
1660 kvm->nr_memslot_pages += new->npages;
1661
1662 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) {
1663 int change = (new_flags & KVM_MEM_LOG_DIRTY_PAGES) ? 1 : -1;
1664 atomic_set(&kvm->nr_memslots_dirty_logging,
1665 atomic_read(&kvm->nr_memslots_dirty_logging) + change);
1666 }
1667
1668 kvm_arch_commit_memory_region(kvm, old, new, change);
1669
1670 switch (change) {
1671 case KVM_MR_CREATE:
1672 /* Nothing more to do. */
1673 break;
1674 case KVM_MR_DELETE:
1675 /* Free the old memslot and all its metadata. */
1676 kvm_free_memslot(kvm, old);
1677 break;
1678 case KVM_MR_MOVE:
1679 case KVM_MR_FLAGS_ONLY:
1680 /*
1681 * Free the dirty bitmap as needed; the below check encompasses
1682 * both the flags and whether a ring buffer is being used)
1683 */
1684 if (old->dirty_bitmap && !new->dirty_bitmap)
1685 kvm_destroy_dirty_bitmap(old);
1686
1687 /*
1688 * The final quirk. Free the detached, old slot, but only its
1689 * memory, not any metadata. Metadata, including arch specific
1690 * data, may be reused by @new.
1691 */
1692 kfree(old);
1693 break;
1694 default:
1695 BUG();
1696 }
1697}
1698
1699/*
1700 * Activate @new, which must be installed in the inactive slots by the caller,
1701 * by swapping the active slots and then propagating @new to @old once @old is
1702 * unreachable and can be safely modified.
1703 *
1704 * With NULL @old this simply adds @new to @active (while swapping the sets).
1705 * With NULL @new this simply removes @old from @active and frees it
1706 * (while also swapping the sets).
1707 */
1708static void kvm_activate_memslot(struct kvm *kvm,
1709 struct kvm_memory_slot *old,
1710 struct kvm_memory_slot *new)
1711{
1712 int as_id = kvm_memslots_get_as_id(old, new);
1713
1714 kvm_swap_active_memslots(kvm, as_id);
1715
1716 /* Propagate the new memslot to the now inactive memslots. */
1717 kvm_replace_memslot(kvm, old, new);
1718}
1719
1720static void kvm_copy_memslot(struct kvm_memory_slot *dest,
1721 const struct kvm_memory_slot *src)
1722{
1723 dest->base_gfn = src->base_gfn;
1724 dest->npages = src->npages;
1725 dest->dirty_bitmap = src->dirty_bitmap;
1726 dest->arch = src->arch;
1727 dest->userspace_addr = src->userspace_addr;
1728 dest->flags = src->flags;
1729 dest->id = src->id;
1730 dest->as_id = src->as_id;
1731}
1732
1733static void kvm_invalidate_memslot(struct kvm *kvm,
1734 struct kvm_memory_slot *old,
1735 struct kvm_memory_slot *invalid_slot)
1736{
1737 /*
1738 * Mark the current slot INVALID. As with all memslot modifications,
1739 * this must be done on an unreachable slot to avoid modifying the
1740 * current slot in the active tree.
1741 */
1742 kvm_copy_memslot(invalid_slot, old);
1743 invalid_slot->flags |= KVM_MEMSLOT_INVALID;
1744 kvm_replace_memslot(kvm, old, invalid_slot);
1745
1746 /*
1747 * Activate the slot that is now marked INVALID, but don't propagate
1748 * the slot to the now inactive slots. The slot is either going to be
1749 * deleted or recreated as a new slot.
1750 */
1751 kvm_swap_active_memslots(kvm, old->as_id);
1752
1753 /*
1754 * From this point no new shadow pages pointing to a deleted, or moved,
1755 * memslot will be created. Validation of sp->gfn happens in:
1756 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1757 * - kvm_is_visible_gfn (mmu_check_root)
1758 */
1759 kvm_arch_flush_shadow_memslot(kvm, old);
1760 kvm_arch_guest_memory_reclaimed(kvm);
1761
1762 /* Was released by kvm_swap_active_memslots(), reacquire. */
1763 mutex_lock(&kvm->slots_arch_lock);
1764
1765 /*
1766 * Copy the arch-specific field of the newly-installed slot back to the
1767 * old slot as the arch data could have changed between releasing
1768 * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock
1769 * above. Writers are required to retrieve memslots *after* acquiring
1770 * slots_arch_lock, thus the active slot's data is guaranteed to be fresh.
1771 */
1772 old->arch = invalid_slot->arch;
1773}
1774
1775static void kvm_create_memslot(struct kvm *kvm,
1776 struct kvm_memory_slot *new)
1777{
1778 /* Add the new memslot to the inactive set and activate. */
1779 kvm_replace_memslot(kvm, NULL, new);
1780 kvm_activate_memslot(kvm, NULL, new);
1781}
1782
1783static void kvm_delete_memslot(struct kvm *kvm,
1784 struct kvm_memory_slot *old,
1785 struct kvm_memory_slot *invalid_slot)
1786{
1787 /*
1788 * Remove the old memslot (in the inactive memslots) by passing NULL as
1789 * the "new" slot, and for the invalid version in the active slots.
1790 */
1791 kvm_replace_memslot(kvm, old, NULL);
1792 kvm_activate_memslot(kvm, invalid_slot, NULL);
1793}
1794
1795static void kvm_move_memslot(struct kvm *kvm,
1796 struct kvm_memory_slot *old,
1797 struct kvm_memory_slot *new,
1798 struct kvm_memory_slot *invalid_slot)
1799{
1800 /*
1801 * Replace the old memslot in the inactive slots, and then swap slots
1802 * and replace the current INVALID with the new as well.
1803 */
1804 kvm_replace_memslot(kvm, old, new);
1805 kvm_activate_memslot(kvm, invalid_slot, new);
1806}
1807
1808static void kvm_update_flags_memslot(struct kvm *kvm,
1809 struct kvm_memory_slot *old,
1810 struct kvm_memory_slot *new)
1811{
1812 /*
1813 * Similar to the MOVE case, but the slot doesn't need to be zapped as
1814 * an intermediate step. Instead, the old memslot is simply replaced
1815 * with a new, updated copy in both memslot sets.
1816 */
1817 kvm_replace_memslot(kvm, old, new);
1818 kvm_activate_memslot(kvm, old, new);
1819}
1820
1821static int kvm_set_memslot(struct kvm *kvm,
1822 struct kvm_memory_slot *old,
1823 struct kvm_memory_slot *new,
1824 enum kvm_mr_change change)
1825{
1826 struct kvm_memory_slot *invalid_slot;
1827 int r;
1828
1829 /*
1830 * Released in kvm_swap_active_memslots().
1831 *
1832 * Must be held from before the current memslots are copied until after
1833 * the new memslots are installed with rcu_assign_pointer, then
1834 * released before the synchronize srcu in kvm_swap_active_memslots().
1835 *
1836 * When modifying memslots outside of the slots_lock, must be held
1837 * before reading the pointer to the current memslots until after all
1838 * changes to those memslots are complete.
1839 *
1840 * These rules ensure that installing new memslots does not lose
1841 * changes made to the previous memslots.
1842 */
1843 mutex_lock(&kvm->slots_arch_lock);
1844
1845 /*
1846 * Invalidate the old slot if it's being deleted or moved. This is
1847 * done prior to actually deleting/moving the memslot to allow vCPUs to
1848 * continue running by ensuring there are no mappings or shadow pages
1849 * for the memslot when it is deleted/moved. Without pre-invalidation
1850 * (and without a lock), a window would exist between effecting the
1851 * delete/move and committing the changes in arch code where KVM or a
1852 * guest could access a non-existent memslot.
1853 *
1854 * Modifications are done on a temporary, unreachable slot. The old
1855 * slot needs to be preserved in case a later step fails and the
1856 * invalidation needs to be reverted.
1857 */
1858 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
1859 invalid_slot = kzalloc(sizeof(*invalid_slot), GFP_KERNEL_ACCOUNT);
1860 if (!invalid_slot) {
1861 mutex_unlock(&kvm->slots_arch_lock);
1862 return -ENOMEM;
1863 }
1864 kvm_invalidate_memslot(kvm, old, invalid_slot);
1865 }
1866
1867 r = kvm_prepare_memory_region(kvm, old, new, change);
1868 if (r) {
1869 /*
1870 * For DELETE/MOVE, revert the above INVALID change. No
1871 * modifications required since the original slot was preserved
1872 * in the inactive slots. Changing the active memslots also
1873 * release slots_arch_lock.
1874 */
1875 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
1876 kvm_activate_memslot(kvm, invalid_slot, old);
1877 kfree(invalid_slot);
1878 } else {
1879 mutex_unlock(&kvm->slots_arch_lock);
1880 }
1881 return r;
1882 }
1883
1884 /*
1885 * For DELETE and MOVE, the working slot is now active as the INVALID
1886 * version of the old slot. MOVE is particularly special as it reuses
1887 * the old slot and returns a copy of the old slot (in working_slot).
1888 * For CREATE, there is no old slot. For DELETE and FLAGS_ONLY, the
1889 * old slot is detached but otherwise preserved.
1890 */
1891 if (change == KVM_MR_CREATE)
1892 kvm_create_memslot(kvm, new);
1893 else if (change == KVM_MR_DELETE)
1894 kvm_delete_memslot(kvm, old, invalid_slot);
1895 else if (change == KVM_MR_MOVE)
1896 kvm_move_memslot(kvm, old, new, invalid_slot);
1897 else if (change == KVM_MR_FLAGS_ONLY)
1898 kvm_update_flags_memslot(kvm, old, new);
1899 else
1900 BUG();
1901
1902 /* Free the temporary INVALID slot used for DELETE and MOVE. */
1903 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
1904 kfree(invalid_slot);
1905
1906 /*
1907 * No need to refresh new->arch, changes after dropping slots_arch_lock
1908 * will directly hit the final, active memslot. Architectures are
1909 * responsible for knowing that new->arch may be stale.
1910 */
1911 kvm_commit_memory_region(kvm, old, new, change);
1912
1913 return 0;
1914}
1915
1916static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id,
1917 gfn_t start, gfn_t end)
1918{
1919 struct kvm_memslot_iter iter;
1920
1921 kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) {
1922 if (iter.slot->id != id)
1923 return true;
1924 }
1925
1926 return false;
1927}
1928
1929/*
1930 * Allocate some memory and give it an address in the guest physical address
1931 * space.
1932 *
1933 * Discontiguous memory is allowed, mostly for framebuffers.
1934 *
1935 * Must be called holding kvm->slots_lock for write.
1936 */
1937int __kvm_set_memory_region(struct kvm *kvm,
1938 const struct kvm_userspace_memory_region2 *mem)
1939{
1940 struct kvm_memory_slot *old, *new;
1941 struct kvm_memslots *slots;
1942 enum kvm_mr_change change;
1943 unsigned long npages;
1944 gfn_t base_gfn;
1945 int as_id, id;
1946 int r;
1947
1948 r = check_memory_region_flags(kvm, mem);
1949 if (r)
1950 return r;
1951
1952 as_id = mem->slot >> 16;
1953 id = (u16)mem->slot;
1954
1955 /* General sanity checks */
1956 if ((mem->memory_size & (PAGE_SIZE - 1)) ||
1957 (mem->memory_size != (unsigned long)mem->memory_size))
1958 return -EINVAL;
1959 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
1960 return -EINVAL;
1961 /* We can read the guest memory with __xxx_user() later on. */
1962 if ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
1963 (mem->userspace_addr != untagged_addr(mem->userspace_addr)) ||
1964 !access_ok((void __user *)(unsigned long)mem->userspace_addr,
1965 mem->memory_size))
1966 return -EINVAL;
1967 if (mem->flags & KVM_MEM_GUEST_MEMFD &&
1968 (mem->guest_memfd_offset & (PAGE_SIZE - 1) ||
1969 mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset))
1970 return -EINVAL;
1971 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
1972 return -EINVAL;
1973 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
1974 return -EINVAL;
1975 if ((mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES)
1976 return -EINVAL;
1977
1978 slots = __kvm_memslots(kvm, as_id);
1979
1980 /*
1981 * Note, the old memslot (and the pointer itself!) may be invalidated
1982 * and/or destroyed by kvm_set_memslot().
1983 */
1984 old = id_to_memslot(slots, id);
1985
1986 if (!mem->memory_size) {
1987 if (!old || !old->npages)
1988 return -EINVAL;
1989
1990 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
1991 return -EIO;
1992
1993 return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);
1994 }
1995
1996 base_gfn = (mem->guest_phys_addr >> PAGE_SHIFT);
1997 npages = (mem->memory_size >> PAGE_SHIFT);
1998
1999 if (!old || !old->npages) {
2000 change = KVM_MR_CREATE;
2001
2002 /*
2003 * To simplify KVM internals, the total number of pages across
2004 * all memslots must fit in an unsigned long.
2005 */
2006 if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
2007 return -EINVAL;
2008 } else { /* Modify an existing slot. */
2009 /* Private memslots are immutable, they can only be deleted. */
2010 if (mem->flags & KVM_MEM_GUEST_MEMFD)
2011 return -EINVAL;
2012 if ((mem->userspace_addr != old->userspace_addr) ||
2013 (npages != old->npages) ||
2014 ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
2015 return -EINVAL;
2016
2017 if (base_gfn != old->base_gfn)
2018 change = KVM_MR_MOVE;
2019 else if (mem->flags != old->flags)
2020 change = KVM_MR_FLAGS_ONLY;
2021 else /* Nothing to change. */
2022 return 0;
2023 }
2024
2025 if ((change == KVM_MR_CREATE || change == KVM_MR_MOVE) &&
2026 kvm_check_memslot_overlap(slots, id, base_gfn, base_gfn + npages))
2027 return -EEXIST;
2028
2029 /* Allocate a slot that will persist in the memslot. */
2030 new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
2031 if (!new)
2032 return -ENOMEM;
2033
2034 new->as_id = as_id;
2035 new->id = id;
2036 new->base_gfn = base_gfn;
2037 new->npages = npages;
2038 new->flags = mem->flags;
2039 new->userspace_addr = mem->userspace_addr;
2040 if (mem->flags & KVM_MEM_GUEST_MEMFD) {
2041 r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
2042 if (r)
2043 goto out;
2044 }
2045
2046 r = kvm_set_memslot(kvm, old, new, change);
2047 if (r)
2048 goto out_unbind;
2049
2050 return 0;
2051
2052out_unbind:
2053 if (mem->flags & KVM_MEM_GUEST_MEMFD)
2054 kvm_gmem_unbind(new);
2055out:
2056 kfree(new);
2057 return r;
2058}
2059EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
2060
2061int kvm_set_memory_region(struct kvm *kvm,
2062 const struct kvm_userspace_memory_region2 *mem)
2063{
2064 int r;
2065
2066 mutex_lock(&kvm->slots_lock);
2067 r = __kvm_set_memory_region(kvm, mem);
2068 mutex_unlock(&kvm->slots_lock);
2069 return r;
2070}
2071EXPORT_SYMBOL_GPL(kvm_set_memory_region);
2072
2073static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
2074 struct kvm_userspace_memory_region2 *mem)
2075{
2076 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
2077 return -EINVAL;
2078
2079 return kvm_set_memory_region(kvm, mem);
2080}
2081
2082#ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
2083/**
2084 * kvm_get_dirty_log - get a snapshot of dirty pages
2085 * @kvm: pointer to kvm instance
2086 * @log: slot id and address to which we copy the log
2087 * @is_dirty: set to '1' if any dirty pages were found
2088 * @memslot: set to the associated memslot, always valid on success
2089 */
2090int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
2091 int *is_dirty, struct kvm_memory_slot **memslot)
2092{
2093 struct kvm_memslots *slots;
2094 int i, as_id, id;
2095 unsigned long n;
2096 unsigned long any = 0;
2097
2098 /* Dirty ring tracking may be exclusive to dirty log tracking */
2099 if (!kvm_use_dirty_bitmap(kvm))
2100 return -ENXIO;
2101
2102 *memslot = NULL;
2103 *is_dirty = 0;
2104
2105 as_id = log->slot >> 16;
2106 id = (u16)log->slot;
2107 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2108 return -EINVAL;
2109
2110 slots = __kvm_memslots(kvm, as_id);
2111 *memslot = id_to_memslot(slots, id);
2112 if (!(*memslot) || !(*memslot)->dirty_bitmap)
2113 return -ENOENT;
2114
2115 kvm_arch_sync_dirty_log(kvm, *memslot);
2116
2117 n = kvm_dirty_bitmap_bytes(*memslot);
2118
2119 for (i = 0; !any && i < n/sizeof(long); ++i)
2120 any = (*memslot)->dirty_bitmap[i];
2121
2122 if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))
2123 return -EFAULT;
2124
2125 if (any)
2126 *is_dirty = 1;
2127 return 0;
2128}
2129EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
2130
2131#else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
2132/**
2133 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
2134 * and reenable dirty page tracking for the corresponding pages.
2135 * @kvm: pointer to kvm instance
2136 * @log: slot id and address to which we copy the log
2137 *
2138 * We need to keep it in mind that VCPU threads can write to the bitmap
2139 * concurrently. So, to avoid losing track of dirty pages we keep the
2140 * following order:
2141 *
2142 * 1. Take a snapshot of the bit and clear it if needed.
2143 * 2. Write protect the corresponding page.
2144 * 3. Copy the snapshot to the userspace.
2145 * 4. Upon return caller flushes TLB's if needed.
2146 *
2147 * Between 2 and 4, the guest may write to the page using the remaining TLB
2148 * entry. This is not a problem because the page is reported dirty using
2149 * the snapshot taken before and step 4 ensures that writes done after
2150 * exiting to userspace will be logged for the next call.
2151 *
2152 */
2153static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
2154{
2155 struct kvm_memslots *slots;
2156 struct kvm_memory_slot *memslot;
2157 int i, as_id, id;
2158 unsigned long n;
2159 unsigned long *dirty_bitmap;
2160 unsigned long *dirty_bitmap_buffer;
2161 bool flush;
2162
2163 /* Dirty ring tracking may be exclusive to dirty log tracking */
2164 if (!kvm_use_dirty_bitmap(kvm))
2165 return -ENXIO;
2166
2167 as_id = log->slot >> 16;
2168 id = (u16)log->slot;
2169 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2170 return -EINVAL;
2171
2172 slots = __kvm_memslots(kvm, as_id);
2173 memslot = id_to_memslot(slots, id);
2174 if (!memslot || !memslot->dirty_bitmap)
2175 return -ENOENT;
2176
2177 dirty_bitmap = memslot->dirty_bitmap;
2178
2179 kvm_arch_sync_dirty_log(kvm, memslot);
2180
2181 n = kvm_dirty_bitmap_bytes(memslot);
2182 flush = false;
2183 if (kvm->manual_dirty_log_protect) {
2184 /*
2185 * Unlike kvm_get_dirty_log, we always return false in *flush,
2186 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There
2187 * is some code duplication between this function and
2188 * kvm_get_dirty_log, but hopefully all architecture
2189 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
2190 * can be eliminated.
2191 */
2192 dirty_bitmap_buffer = dirty_bitmap;
2193 } else {
2194 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
2195 memset(dirty_bitmap_buffer, 0, n);
2196
2197 KVM_MMU_LOCK(kvm);
2198 for (i = 0; i < n / sizeof(long); i++) {
2199 unsigned long mask;
2200 gfn_t offset;
2201
2202 if (!dirty_bitmap[i])
2203 continue;
2204
2205 flush = true;
2206 mask = xchg(&dirty_bitmap[i], 0);
2207 dirty_bitmap_buffer[i] = mask;
2208
2209 offset = i * BITS_PER_LONG;
2210 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
2211 offset, mask);
2212 }
2213 KVM_MMU_UNLOCK(kvm);
2214 }
2215
2216 if (flush)
2217 kvm_flush_remote_tlbs_memslot(kvm, memslot);
2218
2219 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
2220 return -EFAULT;
2221 return 0;
2222}
2223
2224
2225/**
2226 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
2227 * @kvm: kvm instance
2228 * @log: slot id and address to which we copy the log
2229 *
2230 * Steps 1-4 below provide general overview of dirty page logging. See
2231 * kvm_get_dirty_log_protect() function description for additional details.
2232 *
2233 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
2234 * always flush the TLB (step 4) even if previous step failed and the dirty
2235 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
2236 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
2237 * writes will be marked dirty for next log read.
2238 *
2239 * 1. Take a snapshot of the bit and clear it if needed.
2240 * 2. Write protect the corresponding page.
2241 * 3. Copy the snapshot to the userspace.
2242 * 4. Flush TLB's if needed.
2243 */
2244static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
2245 struct kvm_dirty_log *log)
2246{
2247 int r;
2248
2249 mutex_lock(&kvm->slots_lock);
2250
2251 r = kvm_get_dirty_log_protect(kvm, log);
2252
2253 mutex_unlock(&kvm->slots_lock);
2254 return r;
2255}
2256
2257/**
2258 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
2259 * and reenable dirty page tracking for the corresponding pages.
2260 * @kvm: pointer to kvm instance
2261 * @log: slot id and address from which to fetch the bitmap of dirty pages
2262 */
2263static int kvm_clear_dirty_log_protect(struct kvm *kvm,
2264 struct kvm_clear_dirty_log *log)
2265{
2266 struct kvm_memslots *slots;
2267 struct kvm_memory_slot *memslot;
2268 int as_id, id;
2269 gfn_t offset;
2270 unsigned long i, n;
2271 unsigned long *dirty_bitmap;
2272 unsigned long *dirty_bitmap_buffer;
2273 bool flush;
2274
2275 /* Dirty ring tracking may be exclusive to dirty log tracking */
2276 if (!kvm_use_dirty_bitmap(kvm))
2277 return -ENXIO;
2278
2279 as_id = log->slot >> 16;
2280 id = (u16)log->slot;
2281 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
2282 return -EINVAL;
2283
2284 if (log->first_page & 63)
2285 return -EINVAL;
2286
2287 slots = __kvm_memslots(kvm, as_id);
2288 memslot = id_to_memslot(slots, id);
2289 if (!memslot || !memslot->dirty_bitmap)
2290 return -ENOENT;
2291
2292 dirty_bitmap = memslot->dirty_bitmap;
2293
2294 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
2295
2296 if (log->first_page > memslot->npages ||
2297 log->num_pages > memslot->npages - log->first_page ||
2298 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
2299 return -EINVAL;
2300
2301 kvm_arch_sync_dirty_log(kvm, memslot);
2302
2303 flush = false;
2304 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
2305 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
2306 return -EFAULT;
2307
2308 KVM_MMU_LOCK(kvm);
2309 for (offset = log->first_page, i = offset / BITS_PER_LONG,
2310 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
2311 i++, offset += BITS_PER_LONG) {
2312 unsigned long mask = *dirty_bitmap_buffer++;
2313 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
2314 if (!mask)
2315 continue;
2316
2317 mask &= atomic_long_fetch_andnot(mask, p);
2318
2319 /*
2320 * mask contains the bits that really have been cleared. This
2321 * never includes any bits beyond the length of the memslot (if
2322 * the length is not aligned to 64 pages), therefore it is not
2323 * a problem if userspace sets them in log->dirty_bitmap.
2324 */
2325 if (mask) {
2326 flush = true;
2327 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
2328 offset, mask);
2329 }
2330 }
2331 KVM_MMU_UNLOCK(kvm);
2332
2333 if (flush)
2334 kvm_flush_remote_tlbs_memslot(kvm, memslot);
2335
2336 return 0;
2337}
2338
2339static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
2340 struct kvm_clear_dirty_log *log)
2341{
2342 int r;
2343
2344 mutex_lock(&kvm->slots_lock);
2345
2346 r = kvm_clear_dirty_log_protect(kvm, log);
2347
2348 mutex_unlock(&kvm->slots_lock);
2349 return r;
2350}
2351#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
2352
2353#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
2354static u64 kvm_supported_mem_attributes(struct kvm *kvm)
2355{
2356 if (!kvm || kvm_arch_has_private_mem(kvm))
2357 return KVM_MEMORY_ATTRIBUTE_PRIVATE;
2358
2359 return 0;
2360}
2361
2362/*
2363 * Returns true if _all_ gfns in the range [@start, @end) have attributes
2364 * such that the bits in @mask match @attrs.
2365 */
2366bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2367 unsigned long mask, unsigned long attrs)
2368{
2369 XA_STATE(xas, &kvm->mem_attr_array, start);
2370 unsigned long index;
2371 void *entry;
2372
2373 mask &= kvm_supported_mem_attributes(kvm);
2374 if (attrs & ~mask)
2375 return false;
2376
2377 if (end == start + 1)
2378 return (kvm_get_memory_attributes(kvm, start) & mask) == attrs;
2379
2380 guard(rcu)();
2381 if (!attrs)
2382 return !xas_find(&xas, end - 1);
2383
2384 for (index = start; index < end; index++) {
2385 do {
2386 entry = xas_next(&xas);
2387 } while (xas_retry(&xas, entry));
2388
2389 if (xas.xa_index != index ||
2390 (xa_to_value(entry) & mask) != attrs)
2391 return false;
2392 }
2393
2394 return true;
2395}
2396
2397static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
2398 struct kvm_mmu_notifier_range *range)
2399{
2400 struct kvm_gfn_range gfn_range;
2401 struct kvm_memory_slot *slot;
2402 struct kvm_memslots *slots;
2403 struct kvm_memslot_iter iter;
2404 bool found_memslot = false;
2405 bool ret = false;
2406 int i;
2407
2408 gfn_range.arg = range->arg;
2409 gfn_range.may_block = range->may_block;
2410
2411 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
2412 slots = __kvm_memslots(kvm, i);
2413
2414 kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) {
2415 slot = iter.slot;
2416 gfn_range.slot = slot;
2417
2418 gfn_range.start = max(range->start, slot->base_gfn);
2419 gfn_range.end = min(range->end, slot->base_gfn + slot->npages);
2420 if (gfn_range.start >= gfn_range.end)
2421 continue;
2422
2423 if (!found_memslot) {
2424 found_memslot = true;
2425 KVM_MMU_LOCK(kvm);
2426 if (!IS_KVM_NULL_FN(range->on_lock))
2427 range->on_lock(kvm);
2428 }
2429
2430 ret |= range->handler(kvm, &gfn_range);
2431 }
2432 }
2433
2434 if (range->flush_on_ret && ret)
2435 kvm_flush_remote_tlbs(kvm);
2436
2437 if (found_memslot)
2438 KVM_MMU_UNLOCK(kvm);
2439}
2440
2441static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
2442 struct kvm_gfn_range *range)
2443{
2444 /*
2445 * Unconditionally add the range to the invalidation set, regardless of
2446 * whether or not the arch callback actually needs to zap SPTEs. E.g.
2447 * if KVM supports RWX attributes in the future and the attributes are
2448 * going from R=>RW, zapping isn't strictly necessary. Unconditionally
2449 * adding the range allows KVM to require that MMU invalidations add at
2450 * least one range between begin() and end(), e.g. allows KVM to detect
2451 * bugs where the add() is missed. Relaxing the rule *might* be safe,
2452 * but it's not obvious that allowing new mappings while the attributes
2453 * are in flux is desirable or worth the complexity.
2454 */
2455 kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
2456
2457 return kvm_arch_pre_set_memory_attributes(kvm, range);
2458}
2459
2460/* Set @attributes for the gfn range [@start, @end). */
2461static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
2462 unsigned long attributes)
2463{
2464 struct kvm_mmu_notifier_range pre_set_range = {
2465 .start = start,
2466 .end = end,
2467 .handler = kvm_pre_set_memory_attributes,
2468 .on_lock = kvm_mmu_invalidate_begin,
2469 .flush_on_ret = true,
2470 .may_block = true,
2471 };
2472 struct kvm_mmu_notifier_range post_set_range = {
2473 .start = start,
2474 .end = end,
2475 .arg.attributes = attributes,
2476 .handler = kvm_arch_post_set_memory_attributes,
2477 .on_lock = kvm_mmu_invalidate_end,
2478 .may_block = true,
2479 };
2480 unsigned long i;
2481 void *entry;
2482 int r = 0;
2483
2484 entry = attributes ? xa_mk_value(attributes) : NULL;
2485
2486 mutex_lock(&kvm->slots_lock);
2487
2488 /* Nothing to do if the entire range as the desired attributes. */
2489 if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes))
2490 goto out_unlock;
2491
2492 /*
2493 * Reserve memory ahead of time to avoid having to deal with failures
2494 * partway through setting the new attributes.
2495 */
2496 for (i = start; i < end; i++) {
2497 r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
2498 if (r)
2499 goto out_unlock;
2500 }
2501
2502 kvm_handle_gfn_range(kvm, &pre_set_range);
2503
2504 for (i = start; i < end; i++) {
2505 r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
2506 GFP_KERNEL_ACCOUNT));
2507 KVM_BUG_ON(r, kvm);
2508 }
2509
2510 kvm_handle_gfn_range(kvm, &post_set_range);
2511
2512out_unlock:
2513 mutex_unlock(&kvm->slots_lock);
2514
2515 return r;
2516}
2517static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
2518 struct kvm_memory_attributes *attrs)
2519{
2520 gfn_t start, end;
2521
2522 /* flags is currently not used. */
2523 if (attrs->flags)
2524 return -EINVAL;
2525 if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
2526 return -EINVAL;
2527 if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
2528 return -EINVAL;
2529 if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size))
2530 return -EINVAL;
2531
2532 start = attrs->address >> PAGE_SHIFT;
2533 end = (attrs->address + attrs->size) >> PAGE_SHIFT;
2534
2535 /*
2536 * xarray tracks data using "unsigned long", and as a result so does
2537 * KVM. For simplicity, supports generic attributes only on 64-bit
2538 * architectures.
2539 */
2540 BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));
2541
2542 return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
2543}
2544#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
2545
2546struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
2547{
2548 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
2549}
2550EXPORT_SYMBOL_GPL(gfn_to_memslot);
2551
2552struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
2553{
2554 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
2555 u64 gen = slots->generation;
2556 struct kvm_memory_slot *slot;
2557
2558 /*
2559 * This also protects against using a memslot from a different address space,
2560 * since different address spaces have different generation numbers.
2561 */
2562 if (unlikely(gen != vcpu->last_used_slot_gen)) {
2563 vcpu->last_used_slot = NULL;
2564 vcpu->last_used_slot_gen = gen;
2565 }
2566
2567 slot = try_get_memslot(vcpu->last_used_slot, gfn);
2568 if (slot)
2569 return slot;
2570
2571 /*
2572 * Fall back to searching all memslots. We purposely use
2573 * search_memslots() instead of __gfn_to_memslot() to avoid
2574 * thrashing the VM-wide last_used_slot in kvm_memslots.
2575 */
2576 slot = search_memslots(slots, gfn, false);
2577 if (slot) {
2578 vcpu->last_used_slot = slot;
2579 return slot;
2580 }
2581
2582 return NULL;
2583}
2584
2585bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
2586{
2587 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
2588
2589 return kvm_is_visible_memslot(memslot);
2590}
2591EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
2592
2593bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
2594{
2595 struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2596
2597 return kvm_is_visible_memslot(memslot);
2598}
2599EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn);
2600
2601unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
2602{
2603 struct vm_area_struct *vma;
2604 unsigned long addr, size;
2605
2606 size = PAGE_SIZE;
2607
2608 addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
2609 if (kvm_is_error_hva(addr))
2610 return PAGE_SIZE;
2611
2612 mmap_read_lock(current->mm);
2613 vma = find_vma(current->mm, addr);
2614 if (!vma)
2615 goto out;
2616
2617 size = vma_kernel_pagesize(vma);
2618
2619out:
2620 mmap_read_unlock(current->mm);
2621
2622 return size;
2623}
2624
2625static bool memslot_is_readonly(const struct kvm_memory_slot *slot)
2626{
2627 return slot->flags & KVM_MEM_READONLY;
2628}
2629
2630static unsigned long __gfn_to_hva_many(const struct kvm_memory_slot *slot, gfn_t gfn,
2631 gfn_t *nr_pages, bool write)
2632{
2633 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
2634 return KVM_HVA_ERR_BAD;
2635
2636 if (memslot_is_readonly(slot) && write)
2637 return KVM_HVA_ERR_RO_BAD;
2638
2639 if (nr_pages)
2640 *nr_pages = slot->npages - (gfn - slot->base_gfn);
2641
2642 return __gfn_to_hva_memslot(slot, gfn);
2643}
2644
2645static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
2646 gfn_t *nr_pages)
2647{
2648 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
2649}
2650
2651unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
2652 gfn_t gfn)
2653{
2654 return gfn_to_hva_many(slot, gfn, NULL);
2655}
2656EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
2657
2658unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
2659{
2660 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
2661}
2662EXPORT_SYMBOL_GPL(gfn_to_hva);
2663
2664unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
2665{
2666 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
2667}
2668EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
2669
2670/*
2671 * Return the hva of a @gfn and the R/W attribute if possible.
2672 *
2673 * @slot: the kvm_memory_slot which contains @gfn
2674 * @gfn: the gfn to be translated
2675 * @writable: used to return the read/write attribute of the @slot if the hva
2676 * is valid and @writable is not NULL
2677 */
2678unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
2679 gfn_t gfn, bool *writable)
2680{
2681 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
2682
2683 if (!kvm_is_error_hva(hva) && writable)
2684 *writable = !memslot_is_readonly(slot);
2685
2686 return hva;
2687}
2688
2689unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
2690{
2691 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
2692
2693 return gfn_to_hva_memslot_prot(slot, gfn, writable);
2694}
2695
2696unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
2697{
2698 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2699
2700 return gfn_to_hva_memslot_prot(slot, gfn, writable);
2701}
2702
2703static bool kvm_is_ad_tracked_page(struct page *page)
2704{
2705 /*
2706 * Per page-flags.h, pages tagged PG_reserved "should in general not be
2707 * touched (e.g. set dirty) except by its owner".
2708 */
2709 return !PageReserved(page);
2710}
2711
2712static void kvm_set_page_dirty(struct page *page)
2713{
2714 if (kvm_is_ad_tracked_page(page))
2715 SetPageDirty(page);
2716}
2717
2718static void kvm_set_page_accessed(struct page *page)
2719{
2720 if (kvm_is_ad_tracked_page(page))
2721 mark_page_accessed(page);
2722}
2723
2724void kvm_release_page_clean(struct page *page)
2725{
2726 if (!page)
2727 return;
2728
2729 kvm_set_page_accessed(page);
2730 put_page(page);
2731}
2732EXPORT_SYMBOL_GPL(kvm_release_page_clean);
2733
2734void kvm_release_page_dirty(struct page *page)
2735{
2736 if (!page)
2737 return;
2738
2739 kvm_set_page_dirty(page);
2740 kvm_release_page_clean(page);
2741}
2742EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
2743
2744static kvm_pfn_t kvm_resolve_pfn(struct kvm_follow_pfn *kfp, struct page *page,
2745 struct follow_pfnmap_args *map, bool writable)
2746{
2747 kvm_pfn_t pfn;
2748
2749 WARN_ON_ONCE(!!page == !!map);
2750
2751 if (kfp->map_writable)
2752 *kfp->map_writable = writable;
2753
2754 if (map)
2755 pfn = map->pfn;
2756 else
2757 pfn = page_to_pfn(page);
2758
2759 *kfp->refcounted_page = page;
2760
2761 return pfn;
2762}
2763
2764/*
2765 * The fast path to get the writable pfn which will be stored in @pfn,
2766 * true indicates success, otherwise false is returned.
2767 */
2768static bool hva_to_pfn_fast(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
2769{
2770 struct page *page;
2771 bool r;
2772
2773 /*
2774 * Try the fast-only path when the caller wants to pin/get the page for
2775 * writing. If the caller only wants to read the page, KVM must go
2776 * down the full, slow path in order to avoid racing an operation that
2777 * breaks Copy-on-Write (CoW), e.g. so that KVM doesn't end up pointing
2778 * at the old, read-only page while mm/ points at a new, writable page.
2779 */
2780 if (!((kfp->flags & FOLL_WRITE) || kfp->map_writable))
2781 return false;
2782
2783 if (kfp->pin)
2784 r = pin_user_pages_fast(kfp->hva, 1, FOLL_WRITE, &page) == 1;
2785 else
2786 r = get_user_page_fast_only(kfp->hva, FOLL_WRITE, &page);
2787
2788 if (r) {
2789 *pfn = kvm_resolve_pfn(kfp, page, NULL, true);
2790 return true;
2791 }
2792
2793 return false;
2794}
2795
2796/*
2797 * The slow path to get the pfn of the specified host virtual address,
2798 * 1 indicates success, -errno is returned if error is detected.
2799 */
2800static int hva_to_pfn_slow(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
2801{
2802 /*
2803 * When a VCPU accesses a page that is not mapped into the secondary
2804 * MMU, we lookup the page using GUP to map it, so the guest VCPU can
2805 * make progress. We always want to honor NUMA hinting faults in that
2806 * case, because GUP usage corresponds to memory accesses from the VCPU.
2807 * Otherwise, we'd not trigger NUMA hinting faults once a page is
2808 * mapped into the secondary MMU and gets accessed by a VCPU.
2809 *
2810 * Note that get_user_page_fast_only() and FOLL_WRITE for now
2811 * implicitly honor NUMA hinting faults and don't need this flag.
2812 */
2813 unsigned int flags = FOLL_HWPOISON | FOLL_HONOR_NUMA_FAULT | kfp->flags;
2814 struct page *page, *wpage;
2815 int npages;
2816
2817 if (kfp->pin)
2818 npages = pin_user_pages_unlocked(kfp->hva, 1, &page, flags);
2819 else
2820 npages = get_user_pages_unlocked(kfp->hva, 1, &page, flags);
2821 if (npages != 1)
2822 return npages;
2823
2824 /*
2825 * Pinning is mutually exclusive with opportunistically mapping a read
2826 * fault as writable, as KVM should never pin pages when mapping memory
2827 * into the guest (pinning is only for direct accesses from KVM).
2828 */
2829 if (WARN_ON_ONCE(kfp->map_writable && kfp->pin))
2830 goto out;
2831
2832 /* map read fault as writable if possible */
2833 if (!(flags & FOLL_WRITE) && kfp->map_writable &&
2834 get_user_page_fast_only(kfp->hva, FOLL_WRITE, &wpage)) {
2835 put_page(page);
2836 page = wpage;
2837 flags |= FOLL_WRITE;
2838 }
2839
2840out:
2841 *pfn = kvm_resolve_pfn(kfp, page, NULL, flags & FOLL_WRITE);
2842 return npages;
2843}
2844
2845static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
2846{
2847 if (unlikely(!(vma->vm_flags & VM_READ)))
2848 return false;
2849
2850 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
2851 return false;
2852
2853 return true;
2854}
2855
2856static int hva_to_pfn_remapped(struct vm_area_struct *vma,
2857 struct kvm_follow_pfn *kfp, kvm_pfn_t *p_pfn)
2858{
2859 struct follow_pfnmap_args args = { .vma = vma, .address = kfp->hva };
2860 bool write_fault = kfp->flags & FOLL_WRITE;
2861 int r;
2862
2863 /*
2864 * Remapped memory cannot be pinned in any meaningful sense. Bail if
2865 * the caller wants to pin the page, i.e. access the page outside of
2866 * MMU notifier protection, and unsafe umappings are disallowed.
2867 */
2868 if (kfp->pin && !allow_unsafe_mappings)
2869 return -EINVAL;
2870
2871 r = follow_pfnmap_start(&args);
2872 if (r) {
2873 /*
2874 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
2875 * not call the fault handler, so do it here.
2876 */
2877 bool unlocked = false;
2878 r = fixup_user_fault(current->mm, kfp->hva,
2879 (write_fault ? FAULT_FLAG_WRITE : 0),
2880 &unlocked);
2881 if (unlocked)
2882 return -EAGAIN;
2883 if (r)
2884 return r;
2885
2886 r = follow_pfnmap_start(&args);
2887 if (r)
2888 return r;
2889 }
2890
2891 if (write_fault && !args.writable) {
2892 *p_pfn = KVM_PFN_ERR_RO_FAULT;
2893 goto out;
2894 }
2895
2896 *p_pfn = kvm_resolve_pfn(kfp, NULL, &args, args.writable);
2897out:
2898 follow_pfnmap_end(&args);
2899 return r;
2900}
2901
2902kvm_pfn_t hva_to_pfn(struct kvm_follow_pfn *kfp)
2903{
2904 struct vm_area_struct *vma;
2905 kvm_pfn_t pfn;
2906 int npages, r;
2907
2908 might_sleep();
2909
2910 if (WARN_ON_ONCE(!kfp->refcounted_page))
2911 return KVM_PFN_ERR_FAULT;
2912
2913 if (hva_to_pfn_fast(kfp, &pfn))
2914 return pfn;
2915
2916 npages = hva_to_pfn_slow(kfp, &pfn);
2917 if (npages == 1)
2918 return pfn;
2919 if (npages == -EINTR || npages == -EAGAIN)
2920 return KVM_PFN_ERR_SIGPENDING;
2921 if (npages == -EHWPOISON)
2922 return KVM_PFN_ERR_HWPOISON;
2923
2924 mmap_read_lock(current->mm);
2925retry:
2926 vma = vma_lookup(current->mm, kfp->hva);
2927
2928 if (vma == NULL)
2929 pfn = KVM_PFN_ERR_FAULT;
2930 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
2931 r = hva_to_pfn_remapped(vma, kfp, &pfn);
2932 if (r == -EAGAIN)
2933 goto retry;
2934 if (r < 0)
2935 pfn = KVM_PFN_ERR_FAULT;
2936 } else {
2937 if ((kfp->flags & FOLL_NOWAIT) &&
2938 vma_is_valid(vma, kfp->flags & FOLL_WRITE))
2939 pfn = KVM_PFN_ERR_NEEDS_IO;
2940 else
2941 pfn = KVM_PFN_ERR_FAULT;
2942 }
2943 mmap_read_unlock(current->mm);
2944 return pfn;
2945}
2946
2947static kvm_pfn_t kvm_follow_pfn(struct kvm_follow_pfn *kfp)
2948{
2949 kfp->hva = __gfn_to_hva_many(kfp->slot, kfp->gfn, NULL,
2950 kfp->flags & FOLL_WRITE);
2951
2952 if (kfp->hva == KVM_HVA_ERR_RO_BAD)
2953 return KVM_PFN_ERR_RO_FAULT;
2954
2955 if (kvm_is_error_hva(kfp->hva))
2956 return KVM_PFN_NOSLOT;
2957
2958 if (memslot_is_readonly(kfp->slot) && kfp->map_writable) {
2959 *kfp->map_writable = false;
2960 kfp->map_writable = NULL;
2961 }
2962
2963 return hva_to_pfn(kfp);
2964}
2965
2966kvm_pfn_t __kvm_faultin_pfn(const struct kvm_memory_slot *slot, gfn_t gfn,
2967 unsigned int foll, bool *writable,
2968 struct page **refcounted_page)
2969{
2970 struct kvm_follow_pfn kfp = {
2971 .slot = slot,
2972 .gfn = gfn,
2973 .flags = foll,
2974 .map_writable = writable,
2975 .refcounted_page = refcounted_page,
2976 };
2977
2978 if (WARN_ON_ONCE(!writable || !refcounted_page))
2979 return KVM_PFN_ERR_FAULT;
2980
2981 *writable = false;
2982 *refcounted_page = NULL;
2983
2984 return kvm_follow_pfn(&kfp);
2985}
2986EXPORT_SYMBOL_GPL(__kvm_faultin_pfn);
2987
2988int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn,
2989 struct page **pages, int nr_pages)
2990{
2991 unsigned long addr;
2992 gfn_t entry = 0;
2993
2994 addr = gfn_to_hva_many(slot, gfn, &entry);
2995 if (kvm_is_error_hva(addr))
2996 return -1;
2997
2998 if (entry < nr_pages)
2999 return 0;
3000
3001 return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages);
3002}
3003EXPORT_SYMBOL_GPL(kvm_prefetch_pages);
3004
3005/*
3006 * Don't use this API unless you are absolutely, positively certain that KVM
3007 * needs to get a struct page, e.g. to pin the page for firmware DMA.
3008 *
3009 * FIXME: Users of this API likely need to FOLL_PIN the page, not just elevate
3010 * its refcount.
3011 */
3012struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write)
3013{
3014 struct page *refcounted_page = NULL;
3015 struct kvm_follow_pfn kfp = {
3016 .slot = gfn_to_memslot(kvm, gfn),
3017 .gfn = gfn,
3018 .flags = write ? FOLL_WRITE : 0,
3019 .refcounted_page = &refcounted_page,
3020 };
3021
3022 (void)kvm_follow_pfn(&kfp);
3023 return refcounted_page;
3024}
3025EXPORT_SYMBOL_GPL(__gfn_to_page);
3026
3027int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
3028 bool writable)
3029{
3030 struct kvm_follow_pfn kfp = {
3031 .slot = gfn_to_memslot(vcpu->kvm, gfn),
3032 .gfn = gfn,
3033 .flags = writable ? FOLL_WRITE : 0,
3034 .refcounted_page = &map->pinned_page,
3035 .pin = true,
3036 };
3037
3038 map->pinned_page = NULL;
3039 map->page = NULL;
3040 map->hva = NULL;
3041 map->gfn = gfn;
3042 map->writable = writable;
3043
3044 map->pfn = kvm_follow_pfn(&kfp);
3045 if (is_error_noslot_pfn(map->pfn))
3046 return -EINVAL;
3047
3048 if (pfn_valid(map->pfn)) {
3049 map->page = pfn_to_page(map->pfn);
3050 map->hva = kmap(map->page);
3051#ifdef CONFIG_HAS_IOMEM
3052 } else {
3053 map->hva = memremap(pfn_to_hpa(map->pfn), PAGE_SIZE, MEMREMAP_WB);
3054#endif
3055 }
3056
3057 return map->hva ? 0 : -EFAULT;
3058}
3059EXPORT_SYMBOL_GPL(__kvm_vcpu_map);
3060
3061void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map)
3062{
3063 if (!map->hva)
3064 return;
3065
3066 if (map->page)
3067 kunmap(map->page);
3068#ifdef CONFIG_HAS_IOMEM
3069 else
3070 memunmap(map->hva);
3071#endif
3072
3073 if (map->writable)
3074 kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
3075
3076 if (map->pinned_page) {
3077 if (map->writable)
3078 kvm_set_page_dirty(map->pinned_page);
3079 kvm_set_page_accessed(map->pinned_page);
3080 unpin_user_page(map->pinned_page);
3081 }
3082
3083 map->hva = NULL;
3084 map->page = NULL;
3085 map->pinned_page = NULL;
3086}
3087EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);
3088
3089static int next_segment(unsigned long len, int offset)
3090{
3091 if (len > PAGE_SIZE - offset)
3092 return PAGE_SIZE - offset;
3093 else
3094 return len;
3095}
3096
3097/* Copy @len bytes from guest memory at '(@gfn * PAGE_SIZE) + @offset' to @data */
3098static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
3099 void *data, int offset, int len)
3100{
3101 int r;
3102 unsigned long addr;
3103
3104 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3105 return -EFAULT;
3106
3107 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
3108 if (kvm_is_error_hva(addr))
3109 return -EFAULT;
3110 r = __copy_from_user(data, (void __user *)addr + offset, len);
3111 if (r)
3112 return -EFAULT;
3113 return 0;
3114}
3115
3116int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
3117 int len)
3118{
3119 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
3120
3121 return __kvm_read_guest_page(slot, gfn, data, offset, len);
3122}
3123EXPORT_SYMBOL_GPL(kvm_read_guest_page);
3124
3125int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
3126 int offset, int len)
3127{
3128 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3129
3130 return __kvm_read_guest_page(slot, gfn, data, offset, len);
3131}
3132EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
3133
3134int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
3135{
3136 gfn_t gfn = gpa >> PAGE_SHIFT;
3137 int seg;
3138 int offset = offset_in_page(gpa);
3139 int ret;
3140
3141 while ((seg = next_segment(len, offset)) != 0) {
3142 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
3143 if (ret < 0)
3144 return ret;
3145 offset = 0;
3146 len -= seg;
3147 data += seg;
3148 ++gfn;
3149 }
3150 return 0;
3151}
3152EXPORT_SYMBOL_GPL(kvm_read_guest);
3153
3154int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
3155{
3156 gfn_t gfn = gpa >> PAGE_SHIFT;
3157 int seg;
3158 int offset = offset_in_page(gpa);
3159 int ret;
3160
3161 while ((seg = next_segment(len, offset)) != 0) {
3162 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
3163 if (ret < 0)
3164 return ret;
3165 offset = 0;
3166 len -= seg;
3167 data += seg;
3168 ++gfn;
3169 }
3170 return 0;
3171}
3172EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
3173
3174static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
3175 void *data, int offset, unsigned long len)
3176{
3177 int r;
3178 unsigned long addr;
3179
3180 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3181 return -EFAULT;
3182
3183 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
3184 if (kvm_is_error_hva(addr))
3185 return -EFAULT;
3186 pagefault_disable();
3187 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
3188 pagefault_enable();
3189 if (r)
3190 return -EFAULT;
3191 return 0;
3192}
3193
3194int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
3195 void *data, unsigned long len)
3196{
3197 gfn_t gfn = gpa >> PAGE_SHIFT;
3198 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3199 int offset = offset_in_page(gpa);
3200
3201 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
3202}
3203EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
3204
3205/* Copy @len bytes from @data into guest memory at '(@gfn * PAGE_SIZE) + @offset' */
3206static int __kvm_write_guest_page(struct kvm *kvm,
3207 struct kvm_memory_slot *memslot, gfn_t gfn,
3208 const void *data, int offset, int len)
3209{
3210 int r;
3211 unsigned long addr;
3212
3213 if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
3214 return -EFAULT;
3215
3216 addr = gfn_to_hva_memslot(memslot, gfn);
3217 if (kvm_is_error_hva(addr))
3218 return -EFAULT;
3219 r = __copy_to_user((void __user *)addr + offset, data, len);
3220 if (r)
3221 return -EFAULT;
3222 mark_page_dirty_in_slot(kvm, memslot, gfn);
3223 return 0;
3224}
3225
3226int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
3227 const void *data, int offset, int len)
3228{
3229 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
3230
3231 return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);
3232}
3233EXPORT_SYMBOL_GPL(kvm_write_guest_page);
3234
3235int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
3236 const void *data, int offset, int len)
3237{
3238 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3239
3240 return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);
3241}
3242EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
3243
3244int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
3245 unsigned long len)
3246{
3247 gfn_t gfn = gpa >> PAGE_SHIFT;
3248 int seg;
3249 int offset = offset_in_page(gpa);
3250 int ret;
3251
3252 while ((seg = next_segment(len, offset)) != 0) {
3253 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
3254 if (ret < 0)
3255 return ret;
3256 offset = 0;
3257 len -= seg;
3258 data += seg;
3259 ++gfn;
3260 }
3261 return 0;
3262}
3263EXPORT_SYMBOL_GPL(kvm_write_guest);
3264
3265int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
3266 unsigned long len)
3267{
3268 gfn_t gfn = gpa >> PAGE_SHIFT;
3269 int seg;
3270 int offset = offset_in_page(gpa);
3271 int ret;
3272
3273 while ((seg = next_segment(len, offset)) != 0) {
3274 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
3275 if (ret < 0)
3276 return ret;
3277 offset = 0;
3278 len -= seg;
3279 data += seg;
3280 ++gfn;
3281 }
3282 return 0;
3283}
3284EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
3285
3286static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
3287 struct gfn_to_hva_cache *ghc,
3288 gpa_t gpa, unsigned long len)
3289{
3290 int offset = offset_in_page(gpa);
3291 gfn_t start_gfn = gpa >> PAGE_SHIFT;
3292 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
3293 gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
3294 gfn_t nr_pages_avail;
3295
3296 /* Update ghc->generation before performing any error checks. */
3297 ghc->generation = slots->generation;
3298
3299 if (start_gfn > end_gfn) {
3300 ghc->hva = KVM_HVA_ERR_BAD;
3301 return -EINVAL;
3302 }
3303
3304 /*
3305 * If the requested region crosses two memslots, we still
3306 * verify that the entire region is valid here.
3307 */
3308 for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
3309 ghc->memslot = __gfn_to_memslot(slots, start_gfn);
3310 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
3311 &nr_pages_avail);
3312 if (kvm_is_error_hva(ghc->hva))
3313 return -EFAULT;
3314 }
3315
3316 /* Use the slow path for cross page reads and writes. */
3317 if (nr_pages_needed == 1)
3318 ghc->hva += offset;
3319 else
3320 ghc->memslot = NULL;
3321
3322 ghc->gpa = gpa;
3323 ghc->len = len;
3324 return 0;
3325}
3326
3327int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3328 gpa_t gpa, unsigned long len)
3329{
3330 struct kvm_memslots *slots = kvm_memslots(kvm);
3331 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
3332}
3333EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
3334
3335int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3336 void *data, unsigned int offset,
3337 unsigned long len)
3338{
3339 struct kvm_memslots *slots = kvm_memslots(kvm);
3340 int r;
3341 gpa_t gpa = ghc->gpa + offset;
3342
3343 if (WARN_ON_ONCE(len + offset > ghc->len))
3344 return -EINVAL;
3345
3346 if (slots->generation != ghc->generation) {
3347 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
3348 return -EFAULT;
3349 }
3350
3351 if (kvm_is_error_hva(ghc->hva))
3352 return -EFAULT;
3353
3354 if (unlikely(!ghc->memslot))
3355 return kvm_write_guest(kvm, gpa, data, len);
3356
3357 r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
3358 if (r)
3359 return -EFAULT;
3360 mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);
3361
3362 return 0;
3363}
3364EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
3365
3366int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3367 void *data, unsigned long len)
3368{
3369 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
3370}
3371EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
3372
3373int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3374 void *data, unsigned int offset,
3375 unsigned long len)
3376{
3377 struct kvm_memslots *slots = kvm_memslots(kvm);
3378 int r;
3379 gpa_t gpa = ghc->gpa + offset;
3380
3381 if (WARN_ON_ONCE(len + offset > ghc->len))
3382 return -EINVAL;
3383
3384 if (slots->generation != ghc->generation) {
3385 if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
3386 return -EFAULT;
3387 }
3388
3389 if (kvm_is_error_hva(ghc->hva))
3390 return -EFAULT;
3391
3392 if (unlikely(!ghc->memslot))
3393 return kvm_read_guest(kvm, gpa, data, len);
3394
3395 r = __copy_from_user(data, (void __user *)ghc->hva + offset, len);
3396 if (r)
3397 return -EFAULT;
3398
3399 return 0;
3400}
3401EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached);
3402
3403int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
3404 void *data, unsigned long len)
3405{
3406 return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
3407}
3408EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
3409
3410int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
3411{
3412 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
3413 gfn_t gfn = gpa >> PAGE_SHIFT;
3414 int seg;
3415 int offset = offset_in_page(gpa);
3416 int ret;
3417
3418 while ((seg = next_segment(len, offset)) != 0) {
3419 ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, seg);
3420 if (ret < 0)
3421 return ret;
3422 offset = 0;
3423 len -= seg;
3424 ++gfn;
3425 }
3426 return 0;
3427}
3428EXPORT_SYMBOL_GPL(kvm_clear_guest);
3429
3430void mark_page_dirty_in_slot(struct kvm *kvm,
3431 const struct kvm_memory_slot *memslot,
3432 gfn_t gfn)
3433{
3434 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
3435
3436#ifdef CONFIG_HAVE_KVM_DIRTY_RING
3437 if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
3438 return;
3439
3440 WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
3441#endif
3442
3443 if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
3444 unsigned long rel_gfn = gfn - memslot->base_gfn;
3445 u32 slot = (memslot->as_id << 16) | memslot->id;
3446
3447 if (kvm->dirty_ring_size && vcpu)
3448 kvm_dirty_ring_push(vcpu, slot, rel_gfn);
3449 else if (memslot->dirty_bitmap)
3450 set_bit_le(rel_gfn, memslot->dirty_bitmap);
3451 }
3452}
3453EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot);
3454
3455void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
3456{
3457 struct kvm_memory_slot *memslot;
3458
3459 memslot = gfn_to_memslot(kvm, gfn);
3460 mark_page_dirty_in_slot(kvm, memslot, gfn);
3461}
3462EXPORT_SYMBOL_GPL(mark_page_dirty);
3463
3464void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
3465{
3466 struct kvm_memory_slot *memslot;
3467
3468 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3469 mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);
3470}
3471EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
3472
3473void kvm_sigset_activate(struct kvm_vcpu *vcpu)
3474{
3475 if (!vcpu->sigset_active)
3476 return;
3477
3478 /*
3479 * This does a lockless modification of ->real_blocked, which is fine
3480 * because, only current can change ->real_blocked and all readers of
3481 * ->real_blocked don't care as long ->real_blocked is always a subset
3482 * of ->blocked.
3483 */
3484 sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked);
3485}
3486
3487void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
3488{
3489 if (!vcpu->sigset_active)
3490 return;
3491
3492 sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL);
3493 sigemptyset(¤t->real_blocked);
3494}
3495
3496static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
3497{
3498 unsigned int old, val, grow, grow_start;
3499
3500 old = val = vcpu->halt_poll_ns;
3501 grow_start = READ_ONCE(halt_poll_ns_grow_start);
3502 grow = READ_ONCE(halt_poll_ns_grow);
3503 if (!grow)
3504 goto out;
3505
3506 val *= grow;
3507 if (val < grow_start)
3508 val = grow_start;
3509
3510 vcpu->halt_poll_ns = val;
3511out:
3512 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
3513}
3514
3515static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
3516{
3517 unsigned int old, val, shrink, grow_start;
3518
3519 old = val = vcpu->halt_poll_ns;
3520 shrink = READ_ONCE(halt_poll_ns_shrink);
3521 grow_start = READ_ONCE(halt_poll_ns_grow_start);
3522 if (shrink == 0)
3523 val = 0;
3524 else
3525 val /= shrink;
3526
3527 if (val < grow_start)
3528 val = 0;
3529
3530 vcpu->halt_poll_ns = val;
3531 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
3532}
3533
3534static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
3535{
3536 int ret = -EINTR;
3537 int idx = srcu_read_lock(&vcpu->kvm->srcu);
3538
3539 if (kvm_arch_vcpu_runnable(vcpu))
3540 goto out;
3541 if (kvm_cpu_has_pending_timer(vcpu))
3542 goto out;
3543 if (signal_pending(current))
3544 goto out;
3545 if (kvm_check_request(KVM_REQ_UNBLOCK, vcpu))
3546 goto out;
3547
3548 ret = 0;
3549out:
3550 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3551 return ret;
3552}
3553
3554/*
3555 * Block the vCPU until the vCPU is runnable, an event arrives, or a signal is
3556 * pending. This is mostly used when halting a vCPU, but may also be used
3557 * directly for other vCPU non-runnable states, e.g. x86's Wait-For-SIPI.
3558 */
3559bool kvm_vcpu_block(struct kvm_vcpu *vcpu)
3560{
3561 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
3562 bool waited = false;
3563
3564 vcpu->stat.generic.blocking = 1;
3565
3566 preempt_disable();
3567 kvm_arch_vcpu_blocking(vcpu);
3568 prepare_to_rcuwait(wait);
3569 preempt_enable();
3570
3571 for (;;) {
3572 set_current_state(TASK_INTERRUPTIBLE);
3573
3574 if (kvm_vcpu_check_block(vcpu) < 0)
3575 break;
3576
3577 waited = true;
3578 schedule();
3579 }
3580
3581 preempt_disable();
3582 finish_rcuwait(wait);
3583 kvm_arch_vcpu_unblocking(vcpu);
3584 preempt_enable();
3585
3586 vcpu->stat.generic.blocking = 0;
3587
3588 return waited;
3589}
3590
3591static inline void update_halt_poll_stats(struct kvm_vcpu *vcpu, ktime_t start,
3592 ktime_t end, bool success)
3593{
3594 struct kvm_vcpu_stat_generic *stats = &vcpu->stat.generic;
3595 u64 poll_ns = ktime_to_ns(ktime_sub(end, start));
3596
3597 ++vcpu->stat.generic.halt_attempted_poll;
3598
3599 if (success) {
3600 ++vcpu->stat.generic.halt_successful_poll;
3601
3602 if (!vcpu_valid_wakeup(vcpu))
3603 ++vcpu->stat.generic.halt_poll_invalid;
3604
3605 stats->halt_poll_success_ns += poll_ns;
3606 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_success_hist, poll_ns);
3607 } else {
3608 stats->halt_poll_fail_ns += poll_ns;
3609 KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_fail_hist, poll_ns);
3610 }
3611}
3612
3613static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu)
3614{
3615 struct kvm *kvm = vcpu->kvm;
3616
3617 if (kvm->override_halt_poll_ns) {
3618 /*
3619 * Ensure kvm->max_halt_poll_ns is not read before
3620 * kvm->override_halt_poll_ns.
3621 *
3622 * Pairs with the smp_wmb() when enabling KVM_CAP_HALT_POLL.
3623 */
3624 smp_rmb();
3625 return READ_ONCE(kvm->max_halt_poll_ns);
3626 }
3627
3628 return READ_ONCE(halt_poll_ns);
3629}
3630
3631/*
3632 * Emulate a vCPU halt condition, e.g. HLT on x86, WFI on arm, etc... If halt
3633 * polling is enabled, busy wait for a short time before blocking to avoid the
3634 * expensive block+unblock sequence if a wake event arrives soon after the vCPU
3635 * is halted.
3636 */
3637void kvm_vcpu_halt(struct kvm_vcpu *vcpu)
3638{
3639 unsigned int max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
3640 bool halt_poll_allowed = !kvm_arch_no_poll(vcpu);
3641 ktime_t start, cur, poll_end;
3642 bool waited = false;
3643 bool do_halt_poll;
3644 u64 halt_ns;
3645
3646 if (vcpu->halt_poll_ns > max_halt_poll_ns)
3647 vcpu->halt_poll_ns = max_halt_poll_ns;
3648
3649 do_halt_poll = halt_poll_allowed && vcpu->halt_poll_ns;
3650
3651 start = cur = poll_end = ktime_get();
3652 if (do_halt_poll) {
3653 ktime_t stop = ktime_add_ns(start, vcpu->halt_poll_ns);
3654
3655 do {
3656 if (kvm_vcpu_check_block(vcpu) < 0)
3657 goto out;
3658 cpu_relax();
3659 poll_end = cur = ktime_get();
3660 } while (kvm_vcpu_can_poll(cur, stop));
3661 }
3662
3663 waited = kvm_vcpu_block(vcpu);
3664
3665 cur = ktime_get();
3666 if (waited) {
3667 vcpu->stat.generic.halt_wait_ns +=
3668 ktime_to_ns(cur) - ktime_to_ns(poll_end);
3669 KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.generic.halt_wait_hist,
3670 ktime_to_ns(cur) - ktime_to_ns(poll_end));
3671 }
3672out:
3673 /* The total time the vCPU was "halted", including polling time. */
3674 halt_ns = ktime_to_ns(cur) - ktime_to_ns(start);
3675
3676 /*
3677 * Note, halt-polling is considered successful so long as the vCPU was
3678 * never actually scheduled out, i.e. even if the wake event arrived
3679 * after of the halt-polling loop itself, but before the full wait.
3680 */
3681 if (do_halt_poll)
3682 update_halt_poll_stats(vcpu, start, poll_end, !waited);
3683
3684 if (halt_poll_allowed) {
3685 /* Recompute the max halt poll time in case it changed. */
3686 max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
3687
3688 if (!vcpu_valid_wakeup(vcpu)) {
3689 shrink_halt_poll_ns(vcpu);
3690 } else if (max_halt_poll_ns) {
3691 if (halt_ns <= vcpu->halt_poll_ns)
3692 ;
3693 /* we had a long block, shrink polling */
3694 else if (vcpu->halt_poll_ns &&
3695 halt_ns > max_halt_poll_ns)
3696 shrink_halt_poll_ns(vcpu);
3697 /* we had a short halt and our poll time is too small */
3698 else if (vcpu->halt_poll_ns < max_halt_poll_ns &&
3699 halt_ns < max_halt_poll_ns)
3700 grow_halt_poll_ns(vcpu);
3701 } else {
3702 vcpu->halt_poll_ns = 0;
3703 }
3704 }
3705
3706 trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu));
3707}
3708EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
3709
3710bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
3711{
3712 if (__kvm_vcpu_wake_up(vcpu)) {
3713 WRITE_ONCE(vcpu->ready, true);
3714 ++vcpu->stat.generic.halt_wakeup;
3715 return true;
3716 }
3717
3718 return false;
3719}
3720EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
3721
3722#ifndef CONFIG_S390
3723/*
3724 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
3725 */
3726void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
3727{
3728 int me, cpu;
3729
3730 if (kvm_vcpu_wake_up(vcpu))
3731 return;
3732
3733 me = get_cpu();
3734 /*
3735 * The only state change done outside the vcpu mutex is IN_GUEST_MODE
3736 * to EXITING_GUEST_MODE. Therefore the moderately expensive "should
3737 * kick" check does not need atomic operations if kvm_vcpu_kick is used
3738 * within the vCPU thread itself.
3739 */
3740 if (vcpu == __this_cpu_read(kvm_running_vcpu)) {
3741 if (vcpu->mode == IN_GUEST_MODE)
3742 WRITE_ONCE(vcpu->mode, EXITING_GUEST_MODE);
3743 goto out;
3744 }
3745
3746 /*
3747 * Note, the vCPU could get migrated to a different pCPU at any point
3748 * after kvm_arch_vcpu_should_kick(), which could result in sending an
3749 * IPI to the previous pCPU. But, that's ok because the purpose of the
3750 * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the
3751 * vCPU also requires it to leave IN_GUEST_MODE.
3752 */
3753 if (kvm_arch_vcpu_should_kick(vcpu)) {
3754 cpu = READ_ONCE(vcpu->cpu);
3755 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
3756 smp_send_reschedule(cpu);
3757 }
3758out:
3759 put_cpu();
3760}
3761EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
3762#endif /* !CONFIG_S390 */
3763
3764int kvm_vcpu_yield_to(struct kvm_vcpu *target)
3765{
3766 struct task_struct *task = NULL;
3767 int ret;
3768
3769 if (!read_trylock(&target->pid_lock))
3770 return 0;
3771
3772 if (target->pid)
3773 task = get_pid_task(target->pid, PIDTYPE_PID);
3774
3775 read_unlock(&target->pid_lock);
3776
3777 if (!task)
3778 return 0;
3779 ret = yield_to(task, 1);
3780 put_task_struct(task);
3781
3782 return ret;
3783}
3784EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
3785
3786/*
3787 * Helper that checks whether a VCPU is eligible for directed yield.
3788 * Most eligible candidate to yield is decided by following heuristics:
3789 *
3790 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
3791 * (preempted lock holder), indicated by @in_spin_loop.
3792 * Set at the beginning and cleared at the end of interception/PLE handler.
3793 *
3794 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
3795 * chance last time (mostly it has become eligible now since we have probably
3796 * yielded to lockholder in last iteration. This is done by toggling
3797 * @dy_eligible each time a VCPU checked for eligibility.)
3798 *
3799 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
3800 * to preempted lock-holder could result in wrong VCPU selection and CPU
3801 * burning. Giving priority for a potential lock-holder increases lock
3802 * progress.
3803 *
3804 * Since algorithm is based on heuristics, accessing another VCPU data without
3805 * locking does not harm. It may result in trying to yield to same VCPU, fail
3806 * and continue with next VCPU and so on.
3807 */
3808static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
3809{
3810#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
3811 bool eligible;
3812
3813 eligible = !vcpu->spin_loop.in_spin_loop ||
3814 vcpu->spin_loop.dy_eligible;
3815
3816 if (vcpu->spin_loop.in_spin_loop)
3817 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
3818
3819 return eligible;
3820#else
3821 return true;
3822#endif
3823}
3824
3825/*
3826 * Unlike kvm_arch_vcpu_runnable, this function is called outside
3827 * a vcpu_load/vcpu_put pair. However, for most architectures
3828 * kvm_arch_vcpu_runnable does not require vcpu_load.
3829 */
3830bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
3831{
3832 return kvm_arch_vcpu_runnable(vcpu);
3833}
3834
3835static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
3836{
3837 if (kvm_arch_dy_runnable(vcpu))
3838 return true;
3839
3840#ifdef CONFIG_KVM_ASYNC_PF
3841 if (!list_empty_careful(&vcpu->async_pf.done))
3842 return true;
3843#endif
3844
3845 return false;
3846}
3847
3848/*
3849 * By default, simply query the target vCPU's current mode when checking if a
3850 * vCPU was preempted in kernel mode. All architectures except x86 (or more
3851 * specifical, except VMX) allow querying whether or not a vCPU is in kernel
3852 * mode even if the vCPU is NOT loaded, i.e. using kvm_arch_vcpu_in_kernel()
3853 * directly for cross-vCPU checks is functionally correct and accurate.
3854 */
3855bool __weak kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)
3856{
3857 return kvm_arch_vcpu_in_kernel(vcpu);
3858}
3859
3860bool __weak kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
3861{
3862 return false;
3863}
3864
3865void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
3866{
3867 int nr_vcpus, start, i, idx, yielded;
3868 struct kvm *kvm = me->kvm;
3869 struct kvm_vcpu *vcpu;
3870 int try = 3;
3871
3872 nr_vcpus = atomic_read(&kvm->online_vcpus);
3873 if (nr_vcpus < 2)
3874 return;
3875
3876 /* Pairs with the smp_wmb() in kvm_vm_ioctl_create_vcpu(). */
3877 smp_rmb();
3878
3879 kvm_vcpu_set_in_spin_loop(me, true);
3880
3881 /*
3882 * The current vCPU ("me") is spinning in kernel mode, i.e. is likely
3883 * waiting for a resource to become available. Attempt to yield to a
3884 * vCPU that is runnable, but not currently running, e.g. because the
3885 * vCPU was preempted by a higher priority task. With luck, the vCPU
3886 * that was preempted is holding a lock or some other resource that the
3887 * current vCPU is waiting to acquire, and yielding to the other vCPU
3888 * will allow it to make forward progress and release the lock (or kick
3889 * the spinning vCPU, etc).
3890 *
3891 * Since KVM has no insight into what exactly the guest is doing,
3892 * approximate a round-robin selection by iterating over all vCPUs,
3893 * starting at the last boosted vCPU. I.e. if N=kvm->last_boosted_vcpu,
3894 * iterate over vCPU[N+1]..vCPU[N-1], wrapping as needed.
3895 *
3896 * Note, this is inherently racy, e.g. if multiple vCPUs are spinning,
3897 * they may all try to yield to the same vCPU(s). But as above, this
3898 * is all best effort due to KVM's lack of visibility into the guest.
3899 */
3900 start = READ_ONCE(kvm->last_boosted_vcpu) + 1;
3901 for (i = 0; i < nr_vcpus; i++) {
3902 idx = (start + i) % nr_vcpus;
3903 if (idx == me->vcpu_idx)
3904 continue;
3905
3906 vcpu = xa_load(&kvm->vcpu_array, idx);
3907 if (!READ_ONCE(vcpu->ready))
3908 continue;
3909 if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu))
3910 continue;
3911
3912 /*
3913 * Treat the target vCPU as being in-kernel if it has a pending
3914 * interrupt, as the vCPU trying to yield may be spinning
3915 * waiting on IPI delivery, i.e. the target vCPU is in-kernel
3916 * for the purposes of directed yield.
3917 */
3918 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
3919 !kvm_arch_dy_has_pending_interrupt(vcpu) &&
3920 !kvm_arch_vcpu_preempted_in_kernel(vcpu))
3921 continue;
3922
3923 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
3924 continue;
3925
3926 yielded = kvm_vcpu_yield_to(vcpu);
3927 if (yielded > 0) {
3928 WRITE_ONCE(kvm->last_boosted_vcpu, i);
3929 break;
3930 } else if (yielded < 0 && !--try) {
3931 break;
3932 }
3933 }
3934 kvm_vcpu_set_in_spin_loop(me, false);
3935
3936 /* Ensure vcpu is not eligible during next spinloop */
3937 kvm_vcpu_set_dy_eligible(me, false);
3938}
3939EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
3940
3941static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff)
3942{
3943#ifdef CONFIG_HAVE_KVM_DIRTY_RING
3944 return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) &&
3945 (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET +
3946 kvm->dirty_ring_size / PAGE_SIZE);
3947#else
3948 return false;
3949#endif
3950}
3951
3952static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
3953{
3954 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
3955 struct page *page;
3956
3957 if (vmf->pgoff == 0)
3958 page = virt_to_page(vcpu->run);
3959#ifdef CONFIG_X86
3960 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
3961 page = virt_to_page(vcpu->arch.pio_data);
3962#endif
3963#ifdef CONFIG_KVM_MMIO
3964 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
3965 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
3966#endif
3967 else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))
3968 page = kvm_dirty_ring_get_page(
3969 &vcpu->dirty_ring,
3970 vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET);
3971 else
3972 return kvm_arch_vcpu_fault(vcpu, vmf);
3973 get_page(page);
3974 vmf->page = page;
3975 return 0;
3976}
3977
3978static const struct vm_operations_struct kvm_vcpu_vm_ops = {
3979 .fault = kvm_vcpu_fault,
3980};
3981
3982static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
3983{
3984 struct kvm_vcpu *vcpu = file->private_data;
3985 unsigned long pages = vma_pages(vma);
3986
3987 if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||
3988 kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&
3989 ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED)))
3990 return -EINVAL;
3991
3992 vma->vm_ops = &kvm_vcpu_vm_ops;
3993 return 0;
3994}
3995
3996static int kvm_vcpu_release(struct inode *inode, struct file *filp)
3997{
3998 struct kvm_vcpu *vcpu = filp->private_data;
3999
4000 kvm_put_kvm(vcpu->kvm);
4001 return 0;
4002}
4003
4004static struct file_operations kvm_vcpu_fops = {
4005 .release = kvm_vcpu_release,
4006 .unlocked_ioctl = kvm_vcpu_ioctl,
4007 .mmap = kvm_vcpu_mmap,
4008 .llseek = noop_llseek,
4009 KVM_COMPAT(kvm_vcpu_compat_ioctl),
4010};
4011
4012/*
4013 * Allocates an inode for the vcpu.
4014 */
4015static int create_vcpu_fd(struct kvm_vcpu *vcpu)
4016{
4017 char name[8 + 1 + ITOA_MAX_LEN + 1];
4018
4019 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
4020 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
4021}
4022
4023#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
4024static int vcpu_get_pid(void *data, u64 *val)
4025{
4026 struct kvm_vcpu *vcpu = data;
4027
4028 read_lock(&vcpu->pid_lock);
4029 *val = pid_nr(vcpu->pid);
4030 read_unlock(&vcpu->pid_lock);
4031 return 0;
4032}
4033
4034DEFINE_SIMPLE_ATTRIBUTE(vcpu_get_pid_fops, vcpu_get_pid, NULL, "%llu\n");
4035
4036static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
4037{
4038 struct dentry *debugfs_dentry;
4039 char dir_name[ITOA_MAX_LEN * 2];
4040
4041 if (!debugfs_initialized())
4042 return;
4043
4044 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
4045 debugfs_dentry = debugfs_create_dir(dir_name,
4046 vcpu->kvm->debugfs_dentry);
4047 debugfs_create_file("pid", 0444, debugfs_dentry, vcpu,
4048 &vcpu_get_pid_fops);
4049
4050 kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry);
4051}
4052#endif
4053
4054/*
4055 * Creates some virtual cpus. Good luck creating more than one.
4056 */
4057static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
4058{
4059 int r;
4060 struct kvm_vcpu *vcpu;
4061 struct page *page;
4062
4063 /*
4064 * KVM tracks vCPU IDs as 'int', be kind to userspace and reject
4065 * too-large values instead of silently truncating.
4066 *
4067 * Ensure KVM_MAX_VCPU_IDS isn't pushed above INT_MAX without first
4068 * changing the storage type (at the very least, IDs should be tracked
4069 * as unsigned ints).
4070 */
4071 BUILD_BUG_ON(KVM_MAX_VCPU_IDS > INT_MAX);
4072 if (id >= KVM_MAX_VCPU_IDS)
4073 return -EINVAL;
4074
4075 mutex_lock(&kvm->lock);
4076 if (kvm->created_vcpus >= kvm->max_vcpus) {
4077 mutex_unlock(&kvm->lock);
4078 return -EINVAL;
4079 }
4080
4081 r = kvm_arch_vcpu_precreate(kvm, id);
4082 if (r) {
4083 mutex_unlock(&kvm->lock);
4084 return r;
4085 }
4086
4087 kvm->created_vcpus++;
4088 mutex_unlock(&kvm->lock);
4089
4090 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
4091 if (!vcpu) {
4092 r = -ENOMEM;
4093 goto vcpu_decrement;
4094 }
4095
4096 BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
4097 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
4098 if (!page) {
4099 r = -ENOMEM;
4100 goto vcpu_free;
4101 }
4102 vcpu->run = page_address(page);
4103
4104 kvm_vcpu_init(vcpu, kvm, id);
4105
4106 r = kvm_arch_vcpu_create(vcpu);
4107 if (r)
4108 goto vcpu_free_run_page;
4109
4110 if (kvm->dirty_ring_size) {
4111 r = kvm_dirty_ring_alloc(&vcpu->dirty_ring,
4112 id, kvm->dirty_ring_size);
4113 if (r)
4114 goto arch_vcpu_destroy;
4115 }
4116
4117 mutex_lock(&kvm->lock);
4118
4119#ifdef CONFIG_LOCKDEP
4120 /* Ensure that lockdep knows vcpu->mutex is taken *inside* kvm->lock */
4121 mutex_lock(&vcpu->mutex);
4122 mutex_unlock(&vcpu->mutex);
4123#endif
4124
4125 if (kvm_get_vcpu_by_id(kvm, id)) {
4126 r = -EEXIST;
4127 goto unlock_vcpu_destroy;
4128 }
4129
4130 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
4131 r = xa_reserve(&kvm->vcpu_array, vcpu->vcpu_idx, GFP_KERNEL_ACCOUNT);
4132 if (r)
4133 goto unlock_vcpu_destroy;
4134
4135 /* Now it's all set up, let userspace reach it */
4136 kvm_get_kvm(kvm);
4137 r = create_vcpu_fd(vcpu);
4138 if (r < 0)
4139 goto kvm_put_xa_release;
4140
4141 if (KVM_BUG_ON(xa_store(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, 0), kvm)) {
4142 r = -EINVAL;
4143 goto kvm_put_xa_release;
4144 }
4145
4146 /*
4147 * Pairs with smp_rmb() in kvm_get_vcpu. Store the vcpu
4148 * pointer before kvm->online_vcpu's incremented value.
4149 */
4150 smp_wmb();
4151 atomic_inc(&kvm->online_vcpus);
4152
4153 mutex_unlock(&kvm->lock);
4154 kvm_arch_vcpu_postcreate(vcpu);
4155 kvm_create_vcpu_debugfs(vcpu);
4156 return r;
4157
4158kvm_put_xa_release:
4159 kvm_put_kvm_no_destroy(kvm);
4160 xa_release(&kvm->vcpu_array, vcpu->vcpu_idx);
4161unlock_vcpu_destroy:
4162 mutex_unlock(&kvm->lock);
4163 kvm_dirty_ring_free(&vcpu->dirty_ring);
4164arch_vcpu_destroy:
4165 kvm_arch_vcpu_destroy(vcpu);
4166vcpu_free_run_page:
4167 free_page((unsigned long)vcpu->run);
4168vcpu_free:
4169 kmem_cache_free(kvm_vcpu_cache, vcpu);
4170vcpu_decrement:
4171 mutex_lock(&kvm->lock);
4172 kvm->created_vcpus--;
4173 mutex_unlock(&kvm->lock);
4174 return r;
4175}
4176
4177static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
4178{
4179 if (sigset) {
4180 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
4181 vcpu->sigset_active = 1;
4182 vcpu->sigset = *sigset;
4183 } else
4184 vcpu->sigset_active = 0;
4185 return 0;
4186}
4187
4188static ssize_t kvm_vcpu_stats_read(struct file *file, char __user *user_buffer,
4189 size_t size, loff_t *offset)
4190{
4191 struct kvm_vcpu *vcpu = file->private_data;
4192
4193 return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header,
4194 &kvm_vcpu_stats_desc[0], &vcpu->stat,
4195 sizeof(vcpu->stat), user_buffer, size, offset);
4196}
4197
4198static int kvm_vcpu_stats_release(struct inode *inode, struct file *file)
4199{
4200 struct kvm_vcpu *vcpu = file->private_data;
4201
4202 kvm_put_kvm(vcpu->kvm);
4203 return 0;
4204}
4205
4206static const struct file_operations kvm_vcpu_stats_fops = {
4207 .owner = THIS_MODULE,
4208 .read = kvm_vcpu_stats_read,
4209 .release = kvm_vcpu_stats_release,
4210 .llseek = noop_llseek,
4211};
4212
4213static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu)
4214{
4215 int fd;
4216 struct file *file;
4217 char name[15 + ITOA_MAX_LEN + 1];
4218
4219 snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id);
4220
4221 fd = get_unused_fd_flags(O_CLOEXEC);
4222 if (fd < 0)
4223 return fd;
4224
4225 file = anon_inode_getfile(name, &kvm_vcpu_stats_fops, vcpu, O_RDONLY);
4226 if (IS_ERR(file)) {
4227 put_unused_fd(fd);
4228 return PTR_ERR(file);
4229 }
4230
4231 kvm_get_kvm(vcpu->kvm);
4232
4233 file->f_mode |= FMODE_PREAD;
4234 fd_install(fd, file);
4235
4236 return fd;
4237}
4238
4239#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
4240static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
4241 struct kvm_pre_fault_memory *range)
4242{
4243 int idx;
4244 long r;
4245 u64 full_size;
4246
4247 if (range->flags)
4248 return -EINVAL;
4249
4250 if (!PAGE_ALIGNED(range->gpa) ||
4251 !PAGE_ALIGNED(range->size) ||
4252 range->gpa + range->size <= range->gpa)
4253 return -EINVAL;
4254
4255 vcpu_load(vcpu);
4256 idx = srcu_read_lock(&vcpu->kvm->srcu);
4257
4258 full_size = range->size;
4259 do {
4260 if (signal_pending(current)) {
4261 r = -EINTR;
4262 break;
4263 }
4264
4265 r = kvm_arch_vcpu_pre_fault_memory(vcpu, range);
4266 if (WARN_ON_ONCE(r == 0 || r == -EIO))
4267 break;
4268
4269 if (r < 0)
4270 break;
4271
4272 range->size -= r;
4273 range->gpa += r;
4274 cond_resched();
4275 } while (range->size);
4276
4277 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4278 vcpu_put(vcpu);
4279
4280 /* Return success if at least one page was mapped successfully. */
4281 return full_size == range->size ? r : 0;
4282}
4283#endif
4284
4285static long kvm_vcpu_ioctl(struct file *filp,
4286 unsigned int ioctl, unsigned long arg)
4287{
4288 struct kvm_vcpu *vcpu = filp->private_data;
4289 void __user *argp = (void __user *)arg;
4290 int r;
4291 struct kvm_fpu *fpu = NULL;
4292 struct kvm_sregs *kvm_sregs = NULL;
4293
4294 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
4295 return -EIO;
4296
4297 if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
4298 return -EINVAL;
4299
4300 /*
4301 * Some architectures have vcpu ioctls that are asynchronous to vcpu
4302 * execution; mutex_lock() would break them.
4303 */
4304 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
4305 if (r != -ENOIOCTLCMD)
4306 return r;
4307
4308 if (mutex_lock_killable(&vcpu->mutex))
4309 return -EINTR;
4310 switch (ioctl) {
4311 case KVM_RUN: {
4312 struct pid *oldpid;
4313 r = -EINVAL;
4314 if (arg)
4315 goto out;
4316
4317 /*
4318 * Note, vcpu->pid is primarily protected by vcpu->mutex. The
4319 * dedicated r/w lock allows other tasks, e.g. other vCPUs, to
4320 * read vcpu->pid while this vCPU is in KVM_RUN, e.g. to yield
4321 * directly to this vCPU
4322 */
4323 oldpid = vcpu->pid;
4324 if (unlikely(oldpid != task_pid(current))) {
4325 /* The thread running this VCPU changed. */
4326 struct pid *newpid;
4327
4328 r = kvm_arch_vcpu_run_pid_change(vcpu);
4329 if (r)
4330 break;
4331
4332 newpid = get_task_pid(current, PIDTYPE_PID);
4333 write_lock(&vcpu->pid_lock);
4334 vcpu->pid = newpid;
4335 write_unlock(&vcpu->pid_lock);
4336
4337 put_pid(oldpid);
4338 }
4339 vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit__unsafe);
4340 r = kvm_arch_vcpu_ioctl_run(vcpu);
4341 vcpu->wants_to_run = false;
4342
4343 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
4344 break;
4345 }
4346 case KVM_GET_REGS: {
4347 struct kvm_regs *kvm_regs;
4348
4349 r = -ENOMEM;
4350 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
4351 if (!kvm_regs)
4352 goto out;
4353 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
4354 if (r)
4355 goto out_free1;
4356 r = -EFAULT;
4357 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
4358 goto out_free1;
4359 r = 0;
4360out_free1:
4361 kfree(kvm_regs);
4362 break;
4363 }
4364 case KVM_SET_REGS: {
4365 struct kvm_regs *kvm_regs;
4366
4367 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
4368 if (IS_ERR(kvm_regs)) {
4369 r = PTR_ERR(kvm_regs);
4370 goto out;
4371 }
4372 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
4373 kfree(kvm_regs);
4374 break;
4375 }
4376 case KVM_GET_SREGS: {
4377 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
4378 r = -ENOMEM;
4379 if (!kvm_sregs)
4380 goto out;
4381 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
4382 if (r)
4383 goto out;
4384 r = -EFAULT;
4385 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
4386 goto out;
4387 r = 0;
4388 break;
4389 }
4390 case KVM_SET_SREGS: {
4391 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
4392 if (IS_ERR(kvm_sregs)) {
4393 r = PTR_ERR(kvm_sregs);
4394 kvm_sregs = NULL;
4395 goto out;
4396 }
4397 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
4398 break;
4399 }
4400 case KVM_GET_MP_STATE: {
4401 struct kvm_mp_state mp_state;
4402
4403 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
4404 if (r)
4405 goto out;
4406 r = -EFAULT;
4407 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
4408 goto out;
4409 r = 0;
4410 break;
4411 }
4412 case KVM_SET_MP_STATE: {
4413 struct kvm_mp_state mp_state;
4414
4415 r = -EFAULT;
4416 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
4417 goto out;
4418 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
4419 break;
4420 }
4421 case KVM_TRANSLATE: {
4422 struct kvm_translation tr;
4423
4424 r = -EFAULT;
4425 if (copy_from_user(&tr, argp, sizeof(tr)))
4426 goto out;
4427 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
4428 if (r)
4429 goto out;
4430 r = -EFAULT;
4431 if (copy_to_user(argp, &tr, sizeof(tr)))
4432 goto out;
4433 r = 0;
4434 break;
4435 }
4436 case KVM_SET_GUEST_DEBUG: {
4437 struct kvm_guest_debug dbg;
4438
4439 r = -EFAULT;
4440 if (copy_from_user(&dbg, argp, sizeof(dbg)))
4441 goto out;
4442 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
4443 break;
4444 }
4445 case KVM_SET_SIGNAL_MASK: {
4446 struct kvm_signal_mask __user *sigmask_arg = argp;
4447 struct kvm_signal_mask kvm_sigmask;
4448 sigset_t sigset, *p;
4449
4450 p = NULL;
4451 if (argp) {
4452 r = -EFAULT;
4453 if (copy_from_user(&kvm_sigmask, argp,
4454 sizeof(kvm_sigmask)))
4455 goto out;
4456 r = -EINVAL;
4457 if (kvm_sigmask.len != sizeof(sigset))
4458 goto out;
4459 r = -EFAULT;
4460 if (copy_from_user(&sigset, sigmask_arg->sigset,
4461 sizeof(sigset)))
4462 goto out;
4463 p = &sigset;
4464 }
4465 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
4466 break;
4467 }
4468 case KVM_GET_FPU: {
4469 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
4470 r = -ENOMEM;
4471 if (!fpu)
4472 goto out;
4473 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
4474 if (r)
4475 goto out;
4476 r = -EFAULT;
4477 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
4478 goto out;
4479 r = 0;
4480 break;
4481 }
4482 case KVM_SET_FPU: {
4483 fpu = memdup_user(argp, sizeof(*fpu));
4484 if (IS_ERR(fpu)) {
4485 r = PTR_ERR(fpu);
4486 fpu = NULL;
4487 goto out;
4488 }
4489 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
4490 break;
4491 }
4492 case KVM_GET_STATS_FD: {
4493 r = kvm_vcpu_ioctl_get_stats_fd(vcpu);
4494 break;
4495 }
4496#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
4497 case KVM_PRE_FAULT_MEMORY: {
4498 struct kvm_pre_fault_memory range;
4499
4500 r = -EFAULT;
4501 if (copy_from_user(&range, argp, sizeof(range)))
4502 break;
4503 r = kvm_vcpu_pre_fault_memory(vcpu, &range);
4504 /* Pass back leftover range. */
4505 if (copy_to_user(argp, &range, sizeof(range)))
4506 r = -EFAULT;
4507 break;
4508 }
4509#endif
4510 default:
4511 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
4512 }
4513out:
4514 mutex_unlock(&vcpu->mutex);
4515 kfree(fpu);
4516 kfree(kvm_sregs);
4517 return r;
4518}
4519
4520#ifdef CONFIG_KVM_COMPAT
4521static long kvm_vcpu_compat_ioctl(struct file *filp,
4522 unsigned int ioctl, unsigned long arg)
4523{
4524 struct kvm_vcpu *vcpu = filp->private_data;
4525 void __user *argp = compat_ptr(arg);
4526 int r;
4527
4528 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
4529 return -EIO;
4530
4531 switch (ioctl) {
4532 case KVM_SET_SIGNAL_MASK: {
4533 struct kvm_signal_mask __user *sigmask_arg = argp;
4534 struct kvm_signal_mask kvm_sigmask;
4535 sigset_t sigset;
4536
4537 if (argp) {
4538 r = -EFAULT;
4539 if (copy_from_user(&kvm_sigmask, argp,
4540 sizeof(kvm_sigmask)))
4541 goto out;
4542 r = -EINVAL;
4543 if (kvm_sigmask.len != sizeof(compat_sigset_t))
4544 goto out;
4545 r = -EFAULT;
4546 if (get_compat_sigset(&sigset,
4547 (compat_sigset_t __user *)sigmask_arg->sigset))
4548 goto out;
4549 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
4550 } else
4551 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
4552 break;
4553 }
4554 default:
4555 r = kvm_vcpu_ioctl(filp, ioctl, arg);
4556 }
4557
4558out:
4559 return r;
4560}
4561#endif
4562
4563static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
4564{
4565 struct kvm_device *dev = filp->private_data;
4566
4567 if (dev->ops->mmap)
4568 return dev->ops->mmap(dev, vma);
4569
4570 return -ENODEV;
4571}
4572
4573static int kvm_device_ioctl_attr(struct kvm_device *dev,
4574 int (*accessor)(struct kvm_device *dev,
4575 struct kvm_device_attr *attr),
4576 unsigned long arg)
4577{
4578 struct kvm_device_attr attr;
4579
4580 if (!accessor)
4581 return -EPERM;
4582
4583 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4584 return -EFAULT;
4585
4586 return accessor(dev, &attr);
4587}
4588
4589static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
4590 unsigned long arg)
4591{
4592 struct kvm_device *dev = filp->private_data;
4593
4594 if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)
4595 return -EIO;
4596
4597 switch (ioctl) {
4598 case KVM_SET_DEVICE_ATTR:
4599 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
4600 case KVM_GET_DEVICE_ATTR:
4601 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
4602 case KVM_HAS_DEVICE_ATTR:
4603 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
4604 default:
4605 if (dev->ops->ioctl)
4606 return dev->ops->ioctl(dev, ioctl, arg);
4607
4608 return -ENOTTY;
4609 }
4610}
4611
4612static int kvm_device_release(struct inode *inode, struct file *filp)
4613{
4614 struct kvm_device *dev = filp->private_data;
4615 struct kvm *kvm = dev->kvm;
4616
4617 if (dev->ops->release) {
4618 mutex_lock(&kvm->lock);
4619 list_del_rcu(&dev->vm_node);
4620 synchronize_rcu();
4621 dev->ops->release(dev);
4622 mutex_unlock(&kvm->lock);
4623 }
4624
4625 kvm_put_kvm(kvm);
4626 return 0;
4627}
4628
4629static struct file_operations kvm_device_fops = {
4630 .unlocked_ioctl = kvm_device_ioctl,
4631 .release = kvm_device_release,
4632 KVM_COMPAT(kvm_device_ioctl),
4633 .mmap = kvm_device_mmap,
4634};
4635
4636struct kvm_device *kvm_device_from_filp(struct file *filp)
4637{
4638 if (filp->f_op != &kvm_device_fops)
4639 return NULL;
4640
4641 return filp->private_data;
4642}
4643
4644static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
4645#ifdef CONFIG_KVM_MPIC
4646 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
4647 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
4648#endif
4649};
4650
4651int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
4652{
4653 if (type >= ARRAY_SIZE(kvm_device_ops_table))
4654 return -ENOSPC;
4655
4656 if (kvm_device_ops_table[type] != NULL)
4657 return -EEXIST;
4658
4659 kvm_device_ops_table[type] = ops;
4660 return 0;
4661}
4662
4663void kvm_unregister_device_ops(u32 type)
4664{
4665 if (kvm_device_ops_table[type] != NULL)
4666 kvm_device_ops_table[type] = NULL;
4667}
4668
4669static int kvm_ioctl_create_device(struct kvm *kvm,
4670 struct kvm_create_device *cd)
4671{
4672 const struct kvm_device_ops *ops;
4673 struct kvm_device *dev;
4674 bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
4675 int type;
4676 int ret;
4677
4678 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
4679 return -ENODEV;
4680
4681 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
4682 ops = kvm_device_ops_table[type];
4683 if (ops == NULL)
4684 return -ENODEV;
4685
4686 if (test)
4687 return 0;
4688
4689 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
4690 if (!dev)
4691 return -ENOMEM;
4692
4693 dev->ops = ops;
4694 dev->kvm = kvm;
4695
4696 mutex_lock(&kvm->lock);
4697 ret = ops->create(dev, type);
4698 if (ret < 0) {
4699 mutex_unlock(&kvm->lock);
4700 kfree(dev);
4701 return ret;
4702 }
4703 list_add_rcu(&dev->vm_node, &kvm->devices);
4704 mutex_unlock(&kvm->lock);
4705
4706 if (ops->init)
4707 ops->init(dev);
4708
4709 kvm_get_kvm(kvm);
4710 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
4711 if (ret < 0) {
4712 kvm_put_kvm_no_destroy(kvm);
4713 mutex_lock(&kvm->lock);
4714 list_del_rcu(&dev->vm_node);
4715 synchronize_rcu();
4716 if (ops->release)
4717 ops->release(dev);
4718 mutex_unlock(&kvm->lock);
4719 if (ops->destroy)
4720 ops->destroy(dev);
4721 return ret;
4722 }
4723
4724 cd->fd = ret;
4725 return 0;
4726}
4727
4728static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
4729{
4730 switch (arg) {
4731 case KVM_CAP_USER_MEMORY:
4732 case KVM_CAP_USER_MEMORY2:
4733 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
4734 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
4735 case KVM_CAP_INTERNAL_ERROR_DATA:
4736#ifdef CONFIG_HAVE_KVM_MSI
4737 case KVM_CAP_SIGNAL_MSI:
4738#endif
4739#ifdef CONFIG_HAVE_KVM_IRQCHIP
4740 case KVM_CAP_IRQFD:
4741#endif
4742 case KVM_CAP_IOEVENTFD_ANY_LENGTH:
4743 case KVM_CAP_CHECK_EXTENSION_VM:
4744 case KVM_CAP_ENABLE_CAP_VM:
4745 case KVM_CAP_HALT_POLL:
4746 return 1;
4747#ifdef CONFIG_KVM_MMIO
4748 case KVM_CAP_COALESCED_MMIO:
4749 return KVM_COALESCED_MMIO_PAGE_OFFSET;
4750 case KVM_CAP_COALESCED_PIO:
4751 return 1;
4752#endif
4753#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
4754 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
4755 return KVM_DIRTY_LOG_MANUAL_CAPS;
4756#endif
4757#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
4758 case KVM_CAP_IRQ_ROUTING:
4759 return KVM_MAX_IRQ_ROUTES;
4760#endif
4761#if KVM_MAX_NR_ADDRESS_SPACES > 1
4762 case KVM_CAP_MULTI_ADDRESS_SPACE:
4763 if (kvm)
4764 return kvm_arch_nr_memslot_as_ids(kvm);
4765 return KVM_MAX_NR_ADDRESS_SPACES;
4766#endif
4767 case KVM_CAP_NR_MEMSLOTS:
4768 return KVM_USER_MEM_SLOTS;
4769 case KVM_CAP_DIRTY_LOG_RING:
4770#ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO
4771 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
4772#else
4773 return 0;
4774#endif
4775 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
4776#ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL
4777 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
4778#else
4779 return 0;
4780#endif
4781#ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
4782 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
4783#endif
4784 case KVM_CAP_BINARY_STATS_FD:
4785 case KVM_CAP_SYSTEM_EVENT_DATA:
4786 case KVM_CAP_DEVICE_CTRL:
4787 return 1;
4788#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
4789 case KVM_CAP_MEMORY_ATTRIBUTES:
4790 return kvm_supported_mem_attributes(kvm);
4791#endif
4792#ifdef CONFIG_KVM_PRIVATE_MEM
4793 case KVM_CAP_GUEST_MEMFD:
4794 return !kvm || kvm_arch_has_private_mem(kvm);
4795#endif
4796 default:
4797 break;
4798 }
4799 return kvm_vm_ioctl_check_extension(kvm, arg);
4800}
4801
4802static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size)
4803{
4804 int r;
4805
4806 if (!KVM_DIRTY_LOG_PAGE_OFFSET)
4807 return -EINVAL;
4808
4809 /* the size should be power of 2 */
4810 if (!size || (size & (size - 1)))
4811 return -EINVAL;
4812
4813 /* Should be bigger to keep the reserved entries, or a page */
4814 if (size < kvm_dirty_ring_get_rsvd_entries() *
4815 sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE)
4816 return -EINVAL;
4817
4818 if (size > KVM_DIRTY_RING_MAX_ENTRIES *
4819 sizeof(struct kvm_dirty_gfn))
4820 return -E2BIG;
4821
4822 /* We only allow it to set once */
4823 if (kvm->dirty_ring_size)
4824 return -EINVAL;
4825
4826 mutex_lock(&kvm->lock);
4827
4828 if (kvm->created_vcpus) {
4829 /* We don't allow to change this value after vcpu created */
4830 r = -EINVAL;
4831 } else {
4832 kvm->dirty_ring_size = size;
4833 r = 0;
4834 }
4835
4836 mutex_unlock(&kvm->lock);
4837 return r;
4838}
4839
4840static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)
4841{
4842 unsigned long i;
4843 struct kvm_vcpu *vcpu;
4844 int cleared = 0;
4845
4846 if (!kvm->dirty_ring_size)
4847 return -EINVAL;
4848
4849 mutex_lock(&kvm->slots_lock);
4850
4851 kvm_for_each_vcpu(i, vcpu, kvm)
4852 cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring);
4853
4854 mutex_unlock(&kvm->slots_lock);
4855
4856 if (cleared)
4857 kvm_flush_remote_tlbs(kvm);
4858
4859 return cleared;
4860}
4861
4862int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
4863 struct kvm_enable_cap *cap)
4864{
4865 return -EINVAL;
4866}
4867
4868bool kvm_are_all_memslots_empty(struct kvm *kvm)
4869{
4870 int i;
4871
4872 lockdep_assert_held(&kvm->slots_lock);
4873
4874 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
4875 if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
4876 return false;
4877 }
4878
4879 return true;
4880}
4881EXPORT_SYMBOL_GPL(kvm_are_all_memslots_empty);
4882
4883static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
4884 struct kvm_enable_cap *cap)
4885{
4886 switch (cap->cap) {
4887#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
4888 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: {
4889 u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE;
4890
4891 if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE)
4892 allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS;
4893
4894 if (cap->flags || (cap->args[0] & ~allowed_options))
4895 return -EINVAL;
4896 kvm->manual_dirty_log_protect = cap->args[0];
4897 return 0;
4898 }
4899#endif
4900 case KVM_CAP_HALT_POLL: {
4901 if (cap->flags || cap->args[0] != (unsigned int)cap->args[0])
4902 return -EINVAL;
4903
4904 kvm->max_halt_poll_ns = cap->args[0];
4905
4906 /*
4907 * Ensure kvm->override_halt_poll_ns does not become visible
4908 * before kvm->max_halt_poll_ns.
4909 *
4910 * Pairs with the smp_rmb() in kvm_vcpu_max_halt_poll_ns().
4911 */
4912 smp_wmb();
4913 kvm->override_halt_poll_ns = true;
4914
4915 return 0;
4916 }
4917 case KVM_CAP_DIRTY_LOG_RING:
4918 case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
4919 if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))
4920 return -EINVAL;
4921
4922 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
4923 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
4924 int r = -EINVAL;
4925
4926 if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
4927 !kvm->dirty_ring_size || cap->flags)
4928 return r;
4929
4930 mutex_lock(&kvm->slots_lock);
4931
4932 /*
4933 * For simplicity, allow enabling ring+bitmap if and only if
4934 * there are no memslots, e.g. to ensure all memslots allocate
4935 * a bitmap after the capability is enabled.
4936 */
4937 if (kvm_are_all_memslots_empty(kvm)) {
4938 kvm->dirty_ring_with_bitmap = true;
4939 r = 0;
4940 }
4941
4942 mutex_unlock(&kvm->slots_lock);
4943
4944 return r;
4945 }
4946 default:
4947 return kvm_vm_ioctl_enable_cap(kvm, cap);
4948 }
4949}
4950
4951static ssize_t kvm_vm_stats_read(struct file *file, char __user *user_buffer,
4952 size_t size, loff_t *offset)
4953{
4954 struct kvm *kvm = file->private_data;
4955
4956 return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,
4957 &kvm_vm_stats_desc[0], &kvm->stat,
4958 sizeof(kvm->stat), user_buffer, size, offset);
4959}
4960
4961static int kvm_vm_stats_release(struct inode *inode, struct file *file)
4962{
4963 struct kvm *kvm = file->private_data;
4964
4965 kvm_put_kvm(kvm);
4966 return 0;
4967}
4968
4969static const struct file_operations kvm_vm_stats_fops = {
4970 .owner = THIS_MODULE,
4971 .read = kvm_vm_stats_read,
4972 .release = kvm_vm_stats_release,
4973 .llseek = noop_llseek,
4974};
4975
4976static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)
4977{
4978 int fd;
4979 struct file *file;
4980
4981 fd = get_unused_fd_flags(O_CLOEXEC);
4982 if (fd < 0)
4983 return fd;
4984
4985 file = anon_inode_getfile("kvm-vm-stats",
4986 &kvm_vm_stats_fops, kvm, O_RDONLY);
4987 if (IS_ERR(file)) {
4988 put_unused_fd(fd);
4989 return PTR_ERR(file);
4990 }
4991
4992 kvm_get_kvm(kvm);
4993
4994 file->f_mode |= FMODE_PREAD;
4995 fd_install(fd, file);
4996
4997 return fd;
4998}
4999
5000#define SANITY_CHECK_MEM_REGION_FIELD(field) \
5001do { \
5002 BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) != \
5003 offsetof(struct kvm_userspace_memory_region2, field)); \
5004 BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) != \
5005 sizeof_field(struct kvm_userspace_memory_region2, field)); \
5006} while (0)
5007
5008static long kvm_vm_ioctl(struct file *filp,
5009 unsigned int ioctl, unsigned long arg)
5010{
5011 struct kvm *kvm = filp->private_data;
5012 void __user *argp = (void __user *)arg;
5013 int r;
5014
5015 if (kvm->mm != current->mm || kvm->vm_dead)
5016 return -EIO;
5017 switch (ioctl) {
5018 case KVM_CREATE_VCPU:
5019 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
5020 break;
5021 case KVM_ENABLE_CAP: {
5022 struct kvm_enable_cap cap;
5023
5024 r = -EFAULT;
5025 if (copy_from_user(&cap, argp, sizeof(cap)))
5026 goto out;
5027 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
5028 break;
5029 }
5030 case KVM_SET_USER_MEMORY_REGION2:
5031 case KVM_SET_USER_MEMORY_REGION: {
5032 struct kvm_userspace_memory_region2 mem;
5033 unsigned long size;
5034
5035 if (ioctl == KVM_SET_USER_MEMORY_REGION) {
5036 /*
5037 * Fields beyond struct kvm_userspace_memory_region shouldn't be
5038 * accessed, but avoid leaking kernel memory in case of a bug.
5039 */
5040 memset(&mem, 0, sizeof(mem));
5041 size = sizeof(struct kvm_userspace_memory_region);
5042 } else {
5043 size = sizeof(struct kvm_userspace_memory_region2);
5044 }
5045
5046 /* Ensure the common parts of the two structs are identical. */
5047 SANITY_CHECK_MEM_REGION_FIELD(slot);
5048 SANITY_CHECK_MEM_REGION_FIELD(flags);
5049 SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr);
5050 SANITY_CHECK_MEM_REGION_FIELD(memory_size);
5051 SANITY_CHECK_MEM_REGION_FIELD(userspace_addr);
5052
5053 r = -EFAULT;
5054 if (copy_from_user(&mem, argp, size))
5055 goto out;
5056
5057 r = -EINVAL;
5058 if (ioctl == KVM_SET_USER_MEMORY_REGION &&
5059 (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS))
5060 goto out;
5061
5062 r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
5063 break;
5064 }
5065 case KVM_GET_DIRTY_LOG: {
5066 struct kvm_dirty_log log;
5067
5068 r = -EFAULT;
5069 if (copy_from_user(&log, argp, sizeof(log)))
5070 goto out;
5071 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
5072 break;
5073 }
5074#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
5075 case KVM_CLEAR_DIRTY_LOG: {
5076 struct kvm_clear_dirty_log log;
5077
5078 r = -EFAULT;
5079 if (copy_from_user(&log, argp, sizeof(log)))
5080 goto out;
5081 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
5082 break;
5083 }
5084#endif
5085#ifdef CONFIG_KVM_MMIO
5086 case KVM_REGISTER_COALESCED_MMIO: {
5087 struct kvm_coalesced_mmio_zone zone;
5088
5089 r = -EFAULT;
5090 if (copy_from_user(&zone, argp, sizeof(zone)))
5091 goto out;
5092 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
5093 break;
5094 }
5095 case KVM_UNREGISTER_COALESCED_MMIO: {
5096 struct kvm_coalesced_mmio_zone zone;
5097
5098 r = -EFAULT;
5099 if (copy_from_user(&zone, argp, sizeof(zone)))
5100 goto out;
5101 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
5102 break;
5103 }
5104#endif
5105 case KVM_IRQFD: {
5106 struct kvm_irqfd data;
5107
5108 r = -EFAULT;
5109 if (copy_from_user(&data, argp, sizeof(data)))
5110 goto out;
5111 r = kvm_irqfd(kvm, &data);
5112 break;
5113 }
5114 case KVM_IOEVENTFD: {
5115 struct kvm_ioeventfd data;
5116
5117 r = -EFAULT;
5118 if (copy_from_user(&data, argp, sizeof(data)))
5119 goto out;
5120 r = kvm_ioeventfd(kvm, &data);
5121 break;
5122 }
5123#ifdef CONFIG_HAVE_KVM_MSI
5124 case KVM_SIGNAL_MSI: {
5125 struct kvm_msi msi;
5126
5127 r = -EFAULT;
5128 if (copy_from_user(&msi, argp, sizeof(msi)))
5129 goto out;
5130 r = kvm_send_userspace_msi(kvm, &msi);
5131 break;
5132 }
5133#endif
5134#ifdef __KVM_HAVE_IRQ_LINE
5135 case KVM_IRQ_LINE_STATUS:
5136 case KVM_IRQ_LINE: {
5137 struct kvm_irq_level irq_event;
5138
5139 r = -EFAULT;
5140 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
5141 goto out;
5142
5143 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
5144 ioctl == KVM_IRQ_LINE_STATUS);
5145 if (r)
5146 goto out;
5147
5148 r = -EFAULT;
5149 if (ioctl == KVM_IRQ_LINE_STATUS) {
5150 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
5151 goto out;
5152 }
5153
5154 r = 0;
5155 break;
5156 }
5157#endif
5158#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
5159 case KVM_SET_GSI_ROUTING: {
5160 struct kvm_irq_routing routing;
5161 struct kvm_irq_routing __user *urouting;
5162 struct kvm_irq_routing_entry *entries = NULL;
5163
5164 r = -EFAULT;
5165 if (copy_from_user(&routing, argp, sizeof(routing)))
5166 goto out;
5167 r = -EINVAL;
5168 if (!kvm_arch_can_set_irq_routing(kvm))
5169 goto out;
5170 if (routing.nr > KVM_MAX_IRQ_ROUTES)
5171 goto out;
5172 if (routing.flags)
5173 goto out;
5174 if (routing.nr) {
5175 urouting = argp;
5176 entries = vmemdup_array_user(urouting->entries,
5177 routing.nr, sizeof(*entries));
5178 if (IS_ERR(entries)) {
5179 r = PTR_ERR(entries);
5180 goto out;
5181 }
5182 }
5183 r = kvm_set_irq_routing(kvm, entries, routing.nr,
5184 routing.flags);
5185 kvfree(entries);
5186 break;
5187 }
5188#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
5189#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
5190 case KVM_SET_MEMORY_ATTRIBUTES: {
5191 struct kvm_memory_attributes attrs;
5192
5193 r = -EFAULT;
5194 if (copy_from_user(&attrs, argp, sizeof(attrs)))
5195 goto out;
5196
5197 r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
5198 break;
5199 }
5200#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
5201 case KVM_CREATE_DEVICE: {
5202 struct kvm_create_device cd;
5203
5204 r = -EFAULT;
5205 if (copy_from_user(&cd, argp, sizeof(cd)))
5206 goto out;
5207
5208 r = kvm_ioctl_create_device(kvm, &cd);
5209 if (r)
5210 goto out;
5211
5212 r = -EFAULT;
5213 if (copy_to_user(argp, &cd, sizeof(cd)))
5214 goto out;
5215
5216 r = 0;
5217 break;
5218 }
5219 case KVM_CHECK_EXTENSION:
5220 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
5221 break;
5222 case KVM_RESET_DIRTY_RINGS:
5223 r = kvm_vm_ioctl_reset_dirty_pages(kvm);
5224 break;
5225 case KVM_GET_STATS_FD:
5226 r = kvm_vm_ioctl_get_stats_fd(kvm);
5227 break;
5228#ifdef CONFIG_KVM_PRIVATE_MEM
5229 case KVM_CREATE_GUEST_MEMFD: {
5230 struct kvm_create_guest_memfd guest_memfd;
5231
5232 r = -EFAULT;
5233 if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd)))
5234 goto out;
5235
5236 r = kvm_gmem_create(kvm, &guest_memfd);
5237 break;
5238 }
5239#endif
5240 default:
5241 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
5242 }
5243out:
5244 return r;
5245}
5246
5247#ifdef CONFIG_KVM_COMPAT
5248struct compat_kvm_dirty_log {
5249 __u32 slot;
5250 __u32 padding1;
5251 union {
5252 compat_uptr_t dirty_bitmap; /* one bit per page */
5253 __u64 padding2;
5254 };
5255};
5256
5257struct compat_kvm_clear_dirty_log {
5258 __u32 slot;
5259 __u32 num_pages;
5260 __u64 first_page;
5261 union {
5262 compat_uptr_t dirty_bitmap; /* one bit per page */
5263 __u64 padding2;
5264 };
5265};
5266
5267long __weak kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
5268 unsigned long arg)
5269{
5270 return -ENOTTY;
5271}
5272
5273static long kvm_vm_compat_ioctl(struct file *filp,
5274 unsigned int ioctl, unsigned long arg)
5275{
5276 struct kvm *kvm = filp->private_data;
5277 int r;
5278
5279 if (kvm->mm != current->mm || kvm->vm_dead)
5280 return -EIO;
5281
5282 r = kvm_arch_vm_compat_ioctl(filp, ioctl, arg);
5283 if (r != -ENOTTY)
5284 return r;
5285
5286 switch (ioctl) {
5287#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
5288 case KVM_CLEAR_DIRTY_LOG: {
5289 struct compat_kvm_clear_dirty_log compat_log;
5290 struct kvm_clear_dirty_log log;
5291
5292 if (copy_from_user(&compat_log, (void __user *)arg,
5293 sizeof(compat_log)))
5294 return -EFAULT;
5295 log.slot = compat_log.slot;
5296 log.num_pages = compat_log.num_pages;
5297 log.first_page = compat_log.first_page;
5298 log.padding2 = compat_log.padding2;
5299 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
5300
5301 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
5302 break;
5303 }
5304#endif
5305 case KVM_GET_DIRTY_LOG: {
5306 struct compat_kvm_dirty_log compat_log;
5307 struct kvm_dirty_log log;
5308
5309 if (copy_from_user(&compat_log, (void __user *)arg,
5310 sizeof(compat_log)))
5311 return -EFAULT;
5312 log.slot = compat_log.slot;
5313 log.padding1 = compat_log.padding1;
5314 log.padding2 = compat_log.padding2;
5315 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
5316
5317 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
5318 break;
5319 }
5320 default:
5321 r = kvm_vm_ioctl(filp, ioctl, arg);
5322 }
5323 return r;
5324}
5325#endif
5326
5327static struct file_operations kvm_vm_fops = {
5328 .release = kvm_vm_release,
5329 .unlocked_ioctl = kvm_vm_ioctl,
5330 .llseek = noop_llseek,
5331 KVM_COMPAT(kvm_vm_compat_ioctl),
5332};
5333
5334bool file_is_kvm(struct file *file)
5335{
5336 return file && file->f_op == &kvm_vm_fops;
5337}
5338EXPORT_SYMBOL_GPL(file_is_kvm);
5339
5340static int kvm_dev_ioctl_create_vm(unsigned long type)
5341{
5342 char fdname[ITOA_MAX_LEN + 1];
5343 int r, fd;
5344 struct kvm *kvm;
5345 struct file *file;
5346
5347 fd = get_unused_fd_flags(O_CLOEXEC);
5348 if (fd < 0)
5349 return fd;
5350
5351 snprintf(fdname, sizeof(fdname), "%d", fd);
5352
5353 kvm = kvm_create_vm(type, fdname);
5354 if (IS_ERR(kvm)) {
5355 r = PTR_ERR(kvm);
5356 goto put_fd;
5357 }
5358
5359 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
5360 if (IS_ERR(file)) {
5361 r = PTR_ERR(file);
5362 goto put_kvm;
5363 }
5364
5365 /*
5366 * Don't call kvm_put_kvm anymore at this point; file->f_op is
5367 * already set, with ->release() being kvm_vm_release(). In error
5368 * cases it will be called by the final fput(file) and will take
5369 * care of doing kvm_put_kvm(kvm).
5370 */
5371 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
5372
5373 fd_install(fd, file);
5374 return fd;
5375
5376put_kvm:
5377 kvm_put_kvm(kvm);
5378put_fd:
5379 put_unused_fd(fd);
5380 return r;
5381}
5382
5383static long kvm_dev_ioctl(struct file *filp,
5384 unsigned int ioctl, unsigned long arg)
5385{
5386 int r = -EINVAL;
5387
5388 switch (ioctl) {
5389 case KVM_GET_API_VERSION:
5390 if (arg)
5391 goto out;
5392 r = KVM_API_VERSION;
5393 break;
5394 case KVM_CREATE_VM:
5395 r = kvm_dev_ioctl_create_vm(arg);
5396 break;
5397 case KVM_CHECK_EXTENSION:
5398 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
5399 break;
5400 case KVM_GET_VCPU_MMAP_SIZE:
5401 if (arg)
5402 goto out;
5403 r = PAGE_SIZE; /* struct kvm_run */
5404#ifdef CONFIG_X86
5405 r += PAGE_SIZE; /* pio data page */
5406#endif
5407#ifdef CONFIG_KVM_MMIO
5408 r += PAGE_SIZE; /* coalesced mmio ring page */
5409#endif
5410 break;
5411 default:
5412 return kvm_arch_dev_ioctl(filp, ioctl, arg);
5413 }
5414out:
5415 return r;
5416}
5417
5418static struct file_operations kvm_chardev_ops = {
5419 .unlocked_ioctl = kvm_dev_ioctl,
5420 .llseek = noop_llseek,
5421 KVM_COMPAT(kvm_dev_ioctl),
5422};
5423
5424static struct miscdevice kvm_dev = {
5425 KVM_MINOR,
5426 "kvm",
5427 &kvm_chardev_ops,
5428};
5429
5430#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
5431static bool enable_virt_at_load = true;
5432module_param(enable_virt_at_load, bool, 0444);
5433
5434__visible bool kvm_rebooting;
5435EXPORT_SYMBOL_GPL(kvm_rebooting);
5436
5437static DEFINE_PER_CPU(bool, virtualization_enabled);
5438static DEFINE_MUTEX(kvm_usage_lock);
5439static int kvm_usage_count;
5440
5441__weak void kvm_arch_enable_virtualization(void)
5442{
5443
5444}
5445
5446__weak void kvm_arch_disable_virtualization(void)
5447{
5448
5449}
5450
5451static int kvm_enable_virtualization_cpu(void)
5452{
5453 if (__this_cpu_read(virtualization_enabled))
5454 return 0;
5455
5456 if (kvm_arch_enable_virtualization_cpu()) {
5457 pr_info("kvm: enabling virtualization on CPU%d failed\n",
5458 raw_smp_processor_id());
5459 return -EIO;
5460 }
5461
5462 __this_cpu_write(virtualization_enabled, true);
5463 return 0;
5464}
5465
5466static int kvm_online_cpu(unsigned int cpu)
5467{
5468 /*
5469 * Abort the CPU online process if hardware virtualization cannot
5470 * be enabled. Otherwise running VMs would encounter unrecoverable
5471 * errors when scheduled to this CPU.
5472 */
5473 return kvm_enable_virtualization_cpu();
5474}
5475
5476static void kvm_disable_virtualization_cpu(void *ign)
5477{
5478 if (!__this_cpu_read(virtualization_enabled))
5479 return;
5480
5481 kvm_arch_disable_virtualization_cpu();
5482
5483 __this_cpu_write(virtualization_enabled, false);
5484}
5485
5486static int kvm_offline_cpu(unsigned int cpu)
5487{
5488 kvm_disable_virtualization_cpu(NULL);
5489 return 0;
5490}
5491
5492static void kvm_shutdown(void)
5493{
5494 /*
5495 * Disable hardware virtualization and set kvm_rebooting to indicate
5496 * that KVM has asynchronously disabled hardware virtualization, i.e.
5497 * that relevant errors and exceptions aren't entirely unexpected.
5498 * Some flavors of hardware virtualization need to be disabled before
5499 * transferring control to firmware (to perform shutdown/reboot), e.g.
5500 * on x86, virtualization can block INIT interrupts, which are used by
5501 * firmware to pull APs back under firmware control. Note, this path
5502 * is used for both shutdown and reboot scenarios, i.e. neither name is
5503 * 100% comprehensive.
5504 */
5505 pr_info("kvm: exiting hardware virtualization\n");
5506 kvm_rebooting = true;
5507 on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1);
5508}
5509
5510static int kvm_suspend(void)
5511{
5512 /*
5513 * Secondary CPUs and CPU hotplug are disabled across the suspend/resume
5514 * callbacks, i.e. no need to acquire kvm_usage_lock to ensure the usage
5515 * count is stable. Assert that kvm_usage_lock is not held to ensure
5516 * the system isn't suspended while KVM is enabling hardware. Hardware
5517 * enabling can be preempted, but the task cannot be frozen until it has
5518 * dropped all locks (userspace tasks are frozen via a fake signal).
5519 */
5520 lockdep_assert_not_held(&kvm_usage_lock);
5521 lockdep_assert_irqs_disabled();
5522
5523 kvm_disable_virtualization_cpu(NULL);
5524 return 0;
5525}
5526
5527static void kvm_resume(void)
5528{
5529 lockdep_assert_not_held(&kvm_usage_lock);
5530 lockdep_assert_irqs_disabled();
5531
5532 WARN_ON_ONCE(kvm_enable_virtualization_cpu());
5533}
5534
5535static struct syscore_ops kvm_syscore_ops = {
5536 .suspend = kvm_suspend,
5537 .resume = kvm_resume,
5538 .shutdown = kvm_shutdown,
5539};
5540
5541static int kvm_enable_virtualization(void)
5542{
5543 int r;
5544
5545 guard(mutex)(&kvm_usage_lock);
5546
5547 if (kvm_usage_count++)
5548 return 0;
5549
5550 kvm_arch_enable_virtualization();
5551
5552 r = cpuhp_setup_state(CPUHP_AP_KVM_ONLINE, "kvm/cpu:online",
5553 kvm_online_cpu, kvm_offline_cpu);
5554 if (r)
5555 goto err_cpuhp;
5556
5557 register_syscore_ops(&kvm_syscore_ops);
5558
5559 /*
5560 * Undo virtualization enabling and bail if the system is going down.
5561 * If userspace initiated a forced reboot, e.g. reboot -f, then it's
5562 * possible for an in-flight operation to enable virtualization after
5563 * syscore_shutdown() is called, i.e. without kvm_shutdown() being
5564 * invoked. Note, this relies on system_state being set _before_
5565 * kvm_shutdown(), e.g. to ensure either kvm_shutdown() is invoked
5566 * or this CPU observes the impending shutdown. Which is why KVM uses
5567 * a syscore ops hook instead of registering a dedicated reboot
5568 * notifier (the latter runs before system_state is updated).
5569 */
5570 if (system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF ||
5571 system_state == SYSTEM_RESTART) {
5572 r = -EBUSY;
5573 goto err_rebooting;
5574 }
5575
5576 return 0;
5577
5578err_rebooting:
5579 unregister_syscore_ops(&kvm_syscore_ops);
5580 cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
5581err_cpuhp:
5582 kvm_arch_disable_virtualization();
5583 --kvm_usage_count;
5584 return r;
5585}
5586
5587static void kvm_disable_virtualization(void)
5588{
5589 guard(mutex)(&kvm_usage_lock);
5590
5591 if (--kvm_usage_count)
5592 return;
5593
5594 unregister_syscore_ops(&kvm_syscore_ops);
5595 cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
5596 kvm_arch_disable_virtualization();
5597}
5598
5599static int kvm_init_virtualization(void)
5600{
5601 if (enable_virt_at_load)
5602 return kvm_enable_virtualization();
5603
5604 return 0;
5605}
5606
5607static void kvm_uninit_virtualization(void)
5608{
5609 if (enable_virt_at_load)
5610 kvm_disable_virtualization();
5611}
5612#else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
5613static int kvm_enable_virtualization(void)
5614{
5615 return 0;
5616}
5617
5618static int kvm_init_virtualization(void)
5619{
5620 return 0;
5621}
5622
5623static void kvm_disable_virtualization(void)
5624{
5625
5626}
5627
5628static void kvm_uninit_virtualization(void)
5629{
5630
5631}
5632#endif /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
5633
5634static void kvm_iodevice_destructor(struct kvm_io_device *dev)
5635{
5636 if (dev->ops->destructor)
5637 dev->ops->destructor(dev);
5638}
5639
5640static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
5641{
5642 int i;
5643
5644 for (i = 0; i < bus->dev_count; i++) {
5645 struct kvm_io_device *pos = bus->range[i].dev;
5646
5647 kvm_iodevice_destructor(pos);
5648 }
5649 kfree(bus);
5650}
5651
5652static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
5653 const struct kvm_io_range *r2)
5654{
5655 gpa_t addr1 = r1->addr;
5656 gpa_t addr2 = r2->addr;
5657
5658 if (addr1 < addr2)
5659 return -1;
5660
5661 /* If r2->len == 0, match the exact address. If r2->len != 0,
5662 * accept any overlapping write. Any order is acceptable for
5663 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
5664 * we process all of them.
5665 */
5666 if (r2->len) {
5667 addr1 += r1->len;
5668 addr2 += r2->len;
5669 }
5670
5671 if (addr1 > addr2)
5672 return 1;
5673
5674 return 0;
5675}
5676
5677static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
5678{
5679 return kvm_io_bus_cmp(p1, p2);
5680}
5681
5682static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
5683 gpa_t addr, int len)
5684{
5685 struct kvm_io_range *range, key;
5686 int off;
5687
5688 key = (struct kvm_io_range) {
5689 .addr = addr,
5690 .len = len,
5691 };
5692
5693 range = bsearch(&key, bus->range, bus->dev_count,
5694 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
5695 if (range == NULL)
5696 return -ENOENT;
5697
5698 off = range - bus->range;
5699
5700 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
5701 off--;
5702
5703 return off;
5704}
5705
5706static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
5707 struct kvm_io_range *range, const void *val)
5708{
5709 int idx;
5710
5711 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
5712 if (idx < 0)
5713 return -EOPNOTSUPP;
5714
5715 while (idx < bus->dev_count &&
5716 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
5717 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
5718 range->len, val))
5719 return idx;
5720 idx++;
5721 }
5722
5723 return -EOPNOTSUPP;
5724}
5725
5726/* kvm_io_bus_write - called under kvm->slots_lock */
5727int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
5728 int len, const void *val)
5729{
5730 struct kvm_io_bus *bus;
5731 struct kvm_io_range range;
5732 int r;
5733
5734 range = (struct kvm_io_range) {
5735 .addr = addr,
5736 .len = len,
5737 };
5738
5739 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5740 if (!bus)
5741 return -ENOMEM;
5742 r = __kvm_io_bus_write(vcpu, bus, &range, val);
5743 return r < 0 ? r : 0;
5744}
5745EXPORT_SYMBOL_GPL(kvm_io_bus_write);
5746
5747/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
5748int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
5749 gpa_t addr, int len, const void *val, long cookie)
5750{
5751 struct kvm_io_bus *bus;
5752 struct kvm_io_range range;
5753
5754 range = (struct kvm_io_range) {
5755 .addr = addr,
5756 .len = len,
5757 };
5758
5759 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5760 if (!bus)
5761 return -ENOMEM;
5762
5763 /* First try the device referenced by cookie. */
5764 if ((cookie >= 0) && (cookie < bus->dev_count) &&
5765 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
5766 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
5767 val))
5768 return cookie;
5769
5770 /*
5771 * cookie contained garbage; fall back to search and return the
5772 * correct cookie value.
5773 */
5774 return __kvm_io_bus_write(vcpu, bus, &range, val);
5775}
5776
5777static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
5778 struct kvm_io_range *range, void *val)
5779{
5780 int idx;
5781
5782 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
5783 if (idx < 0)
5784 return -EOPNOTSUPP;
5785
5786 while (idx < bus->dev_count &&
5787 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
5788 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
5789 range->len, val))
5790 return idx;
5791 idx++;
5792 }
5793
5794 return -EOPNOTSUPP;
5795}
5796
5797/* kvm_io_bus_read - called under kvm->slots_lock */
5798int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
5799 int len, void *val)
5800{
5801 struct kvm_io_bus *bus;
5802 struct kvm_io_range range;
5803 int r;
5804
5805 range = (struct kvm_io_range) {
5806 .addr = addr,
5807 .len = len,
5808 };
5809
5810 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
5811 if (!bus)
5812 return -ENOMEM;
5813 r = __kvm_io_bus_read(vcpu, bus, &range, val);
5814 return r < 0 ? r : 0;
5815}
5816
5817int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
5818 int len, struct kvm_io_device *dev)
5819{
5820 int i;
5821 struct kvm_io_bus *new_bus, *bus;
5822 struct kvm_io_range range;
5823
5824 lockdep_assert_held(&kvm->slots_lock);
5825
5826 bus = kvm_get_bus(kvm, bus_idx);
5827 if (!bus)
5828 return -ENOMEM;
5829
5830 /* exclude ioeventfd which is limited by maximum fd */
5831 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
5832 return -ENOSPC;
5833
5834 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
5835 GFP_KERNEL_ACCOUNT);
5836 if (!new_bus)
5837 return -ENOMEM;
5838
5839 range = (struct kvm_io_range) {
5840 .addr = addr,
5841 .len = len,
5842 .dev = dev,
5843 };
5844
5845 for (i = 0; i < bus->dev_count; i++)
5846 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
5847 break;
5848
5849 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
5850 new_bus->dev_count++;
5851 new_bus->range[i] = range;
5852 memcpy(new_bus->range + i + 1, bus->range + i,
5853 (bus->dev_count - i) * sizeof(struct kvm_io_range));
5854 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
5855 synchronize_srcu_expedited(&kvm->srcu);
5856 kfree(bus);
5857
5858 return 0;
5859}
5860
5861int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
5862 struct kvm_io_device *dev)
5863{
5864 int i;
5865 struct kvm_io_bus *new_bus, *bus;
5866
5867 lockdep_assert_held(&kvm->slots_lock);
5868
5869 bus = kvm_get_bus(kvm, bus_idx);
5870 if (!bus)
5871 return 0;
5872
5873 for (i = 0; i < bus->dev_count; i++) {
5874 if (bus->range[i].dev == dev) {
5875 break;
5876 }
5877 }
5878
5879 if (i == bus->dev_count)
5880 return 0;
5881
5882 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
5883 GFP_KERNEL_ACCOUNT);
5884 if (new_bus) {
5885 memcpy(new_bus, bus, struct_size(bus, range, i));
5886 new_bus->dev_count--;
5887 memcpy(new_bus->range + i, bus->range + i + 1,
5888 flex_array_size(new_bus, range, new_bus->dev_count - i));
5889 }
5890
5891 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
5892 synchronize_srcu_expedited(&kvm->srcu);
5893
5894 /*
5895 * If NULL bus is installed, destroy the old bus, including all the
5896 * attached devices. Otherwise, destroy the caller's device only.
5897 */
5898 if (!new_bus) {
5899 pr_err("kvm: failed to shrink bus, removing it completely\n");
5900 kvm_io_bus_destroy(bus);
5901 return -ENOMEM;
5902 }
5903
5904 kvm_iodevice_destructor(dev);
5905 kfree(bus);
5906 return 0;
5907}
5908
5909struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
5910 gpa_t addr)
5911{
5912 struct kvm_io_bus *bus;
5913 int dev_idx, srcu_idx;
5914 struct kvm_io_device *iodev = NULL;
5915
5916 srcu_idx = srcu_read_lock(&kvm->srcu);
5917
5918 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
5919 if (!bus)
5920 goto out_unlock;
5921
5922 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
5923 if (dev_idx < 0)
5924 goto out_unlock;
5925
5926 iodev = bus->range[dev_idx].dev;
5927
5928out_unlock:
5929 srcu_read_unlock(&kvm->srcu, srcu_idx);
5930
5931 return iodev;
5932}
5933EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
5934
5935static int kvm_debugfs_open(struct inode *inode, struct file *file,
5936 int (*get)(void *, u64 *), int (*set)(void *, u64),
5937 const char *fmt)
5938{
5939 int ret;
5940 struct kvm_stat_data *stat_data = inode->i_private;
5941
5942 /*
5943 * The debugfs files are a reference to the kvm struct which
5944 * is still valid when kvm_destroy_vm is called. kvm_get_kvm_safe
5945 * avoids the race between open and the removal of the debugfs directory.
5946 */
5947 if (!kvm_get_kvm_safe(stat_data->kvm))
5948 return -ENOENT;
5949
5950 ret = simple_attr_open(inode, file, get,
5951 kvm_stats_debugfs_mode(stat_data->desc) & 0222
5952 ? set : NULL, fmt);
5953 if (ret)
5954 kvm_put_kvm(stat_data->kvm);
5955
5956 return ret;
5957}
5958
5959static int kvm_debugfs_release(struct inode *inode, struct file *file)
5960{
5961 struct kvm_stat_data *stat_data = inode->i_private;
5962
5963 simple_attr_release(inode, file);
5964 kvm_put_kvm(stat_data->kvm);
5965
5966 return 0;
5967}
5968
5969static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
5970{
5971 *val = *(u64 *)((void *)(&kvm->stat) + offset);
5972
5973 return 0;
5974}
5975
5976static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
5977{
5978 *(u64 *)((void *)(&kvm->stat) + offset) = 0;
5979
5980 return 0;
5981}
5982
5983static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
5984{
5985 unsigned long i;
5986 struct kvm_vcpu *vcpu;
5987
5988 *val = 0;
5989
5990 kvm_for_each_vcpu(i, vcpu, kvm)
5991 *val += *(u64 *)((void *)(&vcpu->stat) + offset);
5992
5993 return 0;
5994}
5995
5996static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
5997{
5998 unsigned long i;
5999 struct kvm_vcpu *vcpu;
6000
6001 kvm_for_each_vcpu(i, vcpu, kvm)
6002 *(u64 *)((void *)(&vcpu->stat) + offset) = 0;
6003
6004 return 0;
6005}
6006
6007static int kvm_stat_data_get(void *data, u64 *val)
6008{
6009 int r = -EFAULT;
6010 struct kvm_stat_data *stat_data = data;
6011
6012 switch (stat_data->kind) {
6013 case KVM_STAT_VM:
6014 r = kvm_get_stat_per_vm(stat_data->kvm,
6015 stat_data->desc->desc.offset, val);
6016 break;
6017 case KVM_STAT_VCPU:
6018 r = kvm_get_stat_per_vcpu(stat_data->kvm,
6019 stat_data->desc->desc.offset, val);
6020 break;
6021 }
6022
6023 return r;
6024}
6025
6026static int kvm_stat_data_clear(void *data, u64 val)
6027{
6028 int r = -EFAULT;
6029 struct kvm_stat_data *stat_data = data;
6030
6031 if (val)
6032 return -EINVAL;
6033
6034 switch (stat_data->kind) {
6035 case KVM_STAT_VM:
6036 r = kvm_clear_stat_per_vm(stat_data->kvm,
6037 stat_data->desc->desc.offset);
6038 break;
6039 case KVM_STAT_VCPU:
6040 r = kvm_clear_stat_per_vcpu(stat_data->kvm,
6041 stat_data->desc->desc.offset);
6042 break;
6043 }
6044
6045 return r;
6046}
6047
6048static int kvm_stat_data_open(struct inode *inode, struct file *file)
6049{
6050 __simple_attr_check_format("%llu\n", 0ull);
6051 return kvm_debugfs_open(inode, file, kvm_stat_data_get,
6052 kvm_stat_data_clear, "%llu\n");
6053}
6054
6055static const struct file_operations stat_fops_per_vm = {
6056 .owner = THIS_MODULE,
6057 .open = kvm_stat_data_open,
6058 .release = kvm_debugfs_release,
6059 .read = simple_attr_read,
6060 .write = simple_attr_write,
6061};
6062
6063static int vm_stat_get(void *_offset, u64 *val)
6064{
6065 unsigned offset = (long)_offset;
6066 struct kvm *kvm;
6067 u64 tmp_val;
6068
6069 *val = 0;
6070 mutex_lock(&kvm_lock);
6071 list_for_each_entry(kvm, &vm_list, vm_list) {
6072 kvm_get_stat_per_vm(kvm, offset, &tmp_val);
6073 *val += tmp_val;
6074 }
6075 mutex_unlock(&kvm_lock);
6076 return 0;
6077}
6078
6079static int vm_stat_clear(void *_offset, u64 val)
6080{
6081 unsigned offset = (long)_offset;
6082 struct kvm *kvm;
6083
6084 if (val)
6085 return -EINVAL;
6086
6087 mutex_lock(&kvm_lock);
6088 list_for_each_entry(kvm, &vm_list, vm_list) {
6089 kvm_clear_stat_per_vm(kvm, offset);
6090 }
6091 mutex_unlock(&kvm_lock);
6092
6093 return 0;
6094}
6095
6096DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
6097DEFINE_SIMPLE_ATTRIBUTE(vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n");
6098
6099static int vcpu_stat_get(void *_offset, u64 *val)
6100{
6101 unsigned offset = (long)_offset;
6102 struct kvm *kvm;
6103 u64 tmp_val;
6104
6105 *val = 0;
6106 mutex_lock(&kvm_lock);
6107 list_for_each_entry(kvm, &vm_list, vm_list) {
6108 kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
6109 *val += tmp_val;
6110 }
6111 mutex_unlock(&kvm_lock);
6112 return 0;
6113}
6114
6115static int vcpu_stat_clear(void *_offset, u64 val)
6116{
6117 unsigned offset = (long)_offset;
6118 struct kvm *kvm;
6119
6120 if (val)
6121 return -EINVAL;
6122
6123 mutex_lock(&kvm_lock);
6124 list_for_each_entry(kvm, &vm_list, vm_list) {
6125 kvm_clear_stat_per_vcpu(kvm, offset);
6126 }
6127 mutex_unlock(&kvm_lock);
6128
6129 return 0;
6130}
6131
6132DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
6133 "%llu\n");
6134DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n");
6135
6136static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
6137{
6138 struct kobj_uevent_env *env;
6139 unsigned long long created, active;
6140
6141 if (!kvm_dev.this_device || !kvm)
6142 return;
6143
6144 mutex_lock(&kvm_lock);
6145 if (type == KVM_EVENT_CREATE_VM) {
6146 kvm_createvm_count++;
6147 kvm_active_vms++;
6148 } else if (type == KVM_EVENT_DESTROY_VM) {
6149 kvm_active_vms--;
6150 }
6151 created = kvm_createvm_count;
6152 active = kvm_active_vms;
6153 mutex_unlock(&kvm_lock);
6154
6155 env = kzalloc(sizeof(*env), GFP_KERNEL);
6156 if (!env)
6157 return;
6158
6159 add_uevent_var(env, "CREATED=%llu", created);
6160 add_uevent_var(env, "COUNT=%llu", active);
6161
6162 if (type == KVM_EVENT_CREATE_VM) {
6163 add_uevent_var(env, "EVENT=create");
6164 kvm->userspace_pid = task_pid_nr(current);
6165 } else if (type == KVM_EVENT_DESTROY_VM) {
6166 add_uevent_var(env, "EVENT=destroy");
6167 }
6168 add_uevent_var(env, "PID=%d", kvm->userspace_pid);
6169
6170 if (!IS_ERR(kvm->debugfs_dentry)) {
6171 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);
6172
6173 if (p) {
6174 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
6175 if (!IS_ERR(tmp))
6176 add_uevent_var(env, "STATS_PATH=%s", tmp);
6177 kfree(p);
6178 }
6179 }
6180 /* no need for checks, since we are adding at most only 5 keys */
6181 env->envp[env->envp_idx++] = NULL;
6182 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
6183 kfree(env);
6184}
6185
6186static void kvm_init_debug(void)
6187{
6188 const struct file_operations *fops;
6189 const struct _kvm_stats_desc *pdesc;
6190 int i;
6191
6192 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
6193
6194 for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
6195 pdesc = &kvm_vm_stats_desc[i];
6196 if (kvm_stats_debugfs_mode(pdesc) & 0222)
6197 fops = &vm_stat_fops;
6198 else
6199 fops = &vm_stat_readonly_fops;
6200 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
6201 kvm_debugfs_dir,
6202 (void *)(long)pdesc->desc.offset, fops);
6203 }
6204
6205 for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
6206 pdesc = &kvm_vcpu_stats_desc[i];
6207 if (kvm_stats_debugfs_mode(pdesc) & 0222)
6208 fops = &vcpu_stat_fops;
6209 else
6210 fops = &vcpu_stat_readonly_fops;
6211 debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
6212 kvm_debugfs_dir,
6213 (void *)(long)pdesc->desc.offset, fops);
6214 }
6215}
6216
6217static inline
6218struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
6219{
6220 return container_of(pn, struct kvm_vcpu, preempt_notifier);
6221}
6222
6223static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
6224{
6225 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
6226
6227 WRITE_ONCE(vcpu->preempted, false);
6228 WRITE_ONCE(vcpu->ready, false);
6229
6230 __this_cpu_write(kvm_running_vcpu, vcpu);
6231 kvm_arch_vcpu_load(vcpu, cpu);
6232
6233 WRITE_ONCE(vcpu->scheduled_out, false);
6234}
6235
6236static void kvm_sched_out(struct preempt_notifier *pn,
6237 struct task_struct *next)
6238{
6239 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
6240
6241 WRITE_ONCE(vcpu->scheduled_out, true);
6242
6243 if (task_is_runnable(current) && vcpu->wants_to_run) {
6244 WRITE_ONCE(vcpu->preempted, true);
6245 WRITE_ONCE(vcpu->ready, true);
6246 }
6247 kvm_arch_vcpu_put(vcpu);
6248 __this_cpu_write(kvm_running_vcpu, NULL);
6249}
6250
6251/**
6252 * kvm_get_running_vcpu - get the vcpu running on the current CPU.
6253 *
6254 * We can disable preemption locally around accessing the per-CPU variable,
6255 * and use the resolved vcpu pointer after enabling preemption again,
6256 * because even if the current thread is migrated to another CPU, reading
6257 * the per-CPU value later will give us the same value as we update the
6258 * per-CPU variable in the preempt notifier handlers.
6259 */
6260struct kvm_vcpu *kvm_get_running_vcpu(void)
6261{
6262 struct kvm_vcpu *vcpu;
6263
6264 preempt_disable();
6265 vcpu = __this_cpu_read(kvm_running_vcpu);
6266 preempt_enable();
6267
6268 return vcpu;
6269}
6270EXPORT_SYMBOL_GPL(kvm_get_running_vcpu);
6271
6272/**
6273 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
6274 */
6275struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
6276{
6277 return &kvm_running_vcpu;
6278}
6279
6280#ifdef CONFIG_GUEST_PERF_EVENTS
6281static unsigned int kvm_guest_state(void)
6282{
6283 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
6284 unsigned int state;
6285
6286 if (!kvm_arch_pmi_in_guest(vcpu))
6287 return 0;
6288
6289 state = PERF_GUEST_ACTIVE;
6290 if (!kvm_arch_vcpu_in_kernel(vcpu))
6291 state |= PERF_GUEST_USER;
6292
6293 return state;
6294}
6295
6296static unsigned long kvm_guest_get_ip(void)
6297{
6298 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
6299
6300 /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */
6301 if (WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)))
6302 return 0;
6303
6304 return kvm_arch_vcpu_get_ip(vcpu);
6305}
6306
6307static struct perf_guest_info_callbacks kvm_guest_cbs = {
6308 .state = kvm_guest_state,
6309 .get_ip = kvm_guest_get_ip,
6310 .handle_intel_pt_intr = NULL,
6311};
6312
6313void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void))
6314{
6315 kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler;
6316 perf_register_guest_info_callbacks(&kvm_guest_cbs);
6317}
6318void kvm_unregister_perf_callbacks(void)
6319{
6320 perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
6321}
6322#endif
6323
6324int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
6325{
6326 int r;
6327 int cpu;
6328
6329 /* A kmem cache lets us meet the alignment requirements of fx_save. */
6330 if (!vcpu_align)
6331 vcpu_align = __alignof__(struct kvm_vcpu);
6332 kvm_vcpu_cache =
6333 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
6334 SLAB_ACCOUNT,
6335 offsetof(struct kvm_vcpu, arch),
6336 offsetofend(struct kvm_vcpu, stats_id)
6337 - offsetof(struct kvm_vcpu, arch),
6338 NULL);
6339 if (!kvm_vcpu_cache)
6340 return -ENOMEM;
6341
6342 for_each_possible_cpu(cpu) {
6343 if (!alloc_cpumask_var_node(&per_cpu(cpu_kick_mask, cpu),
6344 GFP_KERNEL, cpu_to_node(cpu))) {
6345 r = -ENOMEM;
6346 goto err_cpu_kick_mask;
6347 }
6348 }
6349
6350 r = kvm_irqfd_init();
6351 if (r)
6352 goto err_irqfd;
6353
6354 r = kvm_async_pf_init();
6355 if (r)
6356 goto err_async_pf;
6357
6358 kvm_chardev_ops.owner = module;
6359 kvm_vm_fops.owner = module;
6360 kvm_vcpu_fops.owner = module;
6361 kvm_device_fops.owner = module;
6362
6363 kvm_preempt_ops.sched_in = kvm_sched_in;
6364 kvm_preempt_ops.sched_out = kvm_sched_out;
6365
6366 kvm_init_debug();
6367
6368 r = kvm_vfio_ops_init();
6369 if (WARN_ON_ONCE(r))
6370 goto err_vfio;
6371
6372 kvm_gmem_init(module);
6373
6374 r = kvm_init_virtualization();
6375 if (r)
6376 goto err_virt;
6377
6378 /*
6379 * Registration _must_ be the very last thing done, as this exposes
6380 * /dev/kvm to userspace, i.e. all infrastructure must be setup!
6381 */
6382 r = misc_register(&kvm_dev);
6383 if (r) {
6384 pr_err("kvm: misc device register failed\n");
6385 goto err_register;
6386 }
6387
6388 return 0;
6389
6390err_register:
6391 kvm_uninit_virtualization();
6392err_virt:
6393 kvm_vfio_ops_exit();
6394err_vfio:
6395 kvm_async_pf_deinit();
6396err_async_pf:
6397 kvm_irqfd_exit();
6398err_irqfd:
6399err_cpu_kick_mask:
6400 for_each_possible_cpu(cpu)
6401 free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
6402 kmem_cache_destroy(kvm_vcpu_cache);
6403 return r;
6404}
6405EXPORT_SYMBOL_GPL(kvm_init);
6406
6407void kvm_exit(void)
6408{
6409 int cpu;
6410
6411 /*
6412 * Note, unregistering /dev/kvm doesn't strictly need to come first,
6413 * fops_get(), a.k.a. try_module_get(), prevents acquiring references
6414 * to KVM while the module is being stopped.
6415 */
6416 misc_deregister(&kvm_dev);
6417
6418 kvm_uninit_virtualization();
6419
6420 debugfs_remove_recursive(kvm_debugfs_dir);
6421 for_each_possible_cpu(cpu)
6422 free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
6423 kmem_cache_destroy(kvm_vcpu_cache);
6424 kvm_vfio_ops_exit();
6425 kvm_async_pf_deinit();
6426 kvm_irqfd_exit();
6427}
6428EXPORT_SYMBOL_GPL(kvm_exit);
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9 *
10 * Authors:
11 * Avi Kivity <avi@qumranet.com>
12 * Yaniv Kamay <yaniv@qumranet.com>
13 *
14 * This work is licensed under the terms of the GNU GPL, version 2. See
15 * the COPYING file in the top-level directory.
16 *
17 */
18
19#include "iodev.h"
20
21#include <linux/kvm_host.h>
22#include <linux/kvm.h>
23#include <linux/module.h>
24#include <linux/errno.h>
25#include <linux/percpu.h>
26#include <linux/mm.h>
27#include <linux/miscdevice.h>
28#include <linux/vmalloc.h>
29#include <linux/reboot.h>
30#include <linux/debugfs.h>
31#include <linux/highmem.h>
32#include <linux/file.h>
33#include <linux/syscore_ops.h>
34#include <linux/cpu.h>
35#include <linux/sched.h>
36#include <linux/cpumask.h>
37#include <linux/smp.h>
38#include <linux/anon_inodes.h>
39#include <linux/profile.h>
40#include <linux/kvm_para.h>
41#include <linux/pagemap.h>
42#include <linux/mman.h>
43#include <linux/swap.h>
44#include <linux/bitops.h>
45#include <linux/spinlock.h>
46#include <linux/compat.h>
47#include <linux/srcu.h>
48#include <linux/hugetlb.h>
49#include <linux/slab.h>
50
51#include <asm/processor.h>
52#include <asm/io.h>
53#include <asm/uaccess.h>
54#include <asm/pgtable.h>
55
56#include "coalesced_mmio.h"
57#include "async_pf.h"
58
59#define CREATE_TRACE_POINTS
60#include <trace/events/kvm.h>
61
62MODULE_AUTHOR("Qumranet");
63MODULE_LICENSE("GPL");
64
65/*
66 * Ordering of locks:
67 *
68 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
69 */
70
71DEFINE_RAW_SPINLOCK(kvm_lock);
72LIST_HEAD(vm_list);
73
74static cpumask_var_t cpus_hardware_enabled;
75static int kvm_usage_count = 0;
76static atomic_t hardware_enable_failed;
77
78struct kmem_cache *kvm_vcpu_cache;
79EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
80
81static __read_mostly struct preempt_ops kvm_preempt_ops;
82
83struct dentry *kvm_debugfs_dir;
84
85static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
86 unsigned long arg);
87#ifdef CONFIG_COMPAT
88static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
89 unsigned long arg);
90#endif
91static int hardware_enable_all(void);
92static void hardware_disable_all(void);
93
94static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
95
96bool kvm_rebooting;
97EXPORT_SYMBOL_GPL(kvm_rebooting);
98
99static bool largepages_enabled = true;
100
101static struct page *hwpoison_page;
102static pfn_t hwpoison_pfn;
103
104struct page *fault_page;
105pfn_t fault_pfn;
106
107inline int kvm_is_mmio_pfn(pfn_t pfn)
108{
109 if (pfn_valid(pfn)) {
110 int reserved;
111 struct page *tail = pfn_to_page(pfn);
112 struct page *head = compound_trans_head(tail);
113 reserved = PageReserved(head);
114 if (head != tail) {
115 /*
116 * "head" is not a dangling pointer
117 * (compound_trans_head takes care of that)
118 * but the hugepage may have been splitted
119 * from under us (and we may not hold a
120 * reference count on the head page so it can
121 * be reused before we run PageReferenced), so
122 * we've to check PageTail before returning
123 * what we just read.
124 */
125 smp_rmb();
126 if (PageTail(tail))
127 return reserved;
128 }
129 return PageReserved(tail);
130 }
131
132 return true;
133}
134
135/*
136 * Switches to specified vcpu, until a matching vcpu_put()
137 */
138void vcpu_load(struct kvm_vcpu *vcpu)
139{
140 int cpu;
141
142 mutex_lock(&vcpu->mutex);
143 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
144 /* The thread running this VCPU changed. */
145 struct pid *oldpid = vcpu->pid;
146 struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
147 rcu_assign_pointer(vcpu->pid, newpid);
148 synchronize_rcu();
149 put_pid(oldpid);
150 }
151 cpu = get_cpu();
152 preempt_notifier_register(&vcpu->preempt_notifier);
153 kvm_arch_vcpu_load(vcpu, cpu);
154 put_cpu();
155}
156
157void vcpu_put(struct kvm_vcpu *vcpu)
158{
159 preempt_disable();
160 kvm_arch_vcpu_put(vcpu);
161 preempt_notifier_unregister(&vcpu->preempt_notifier);
162 preempt_enable();
163 mutex_unlock(&vcpu->mutex);
164}
165
166static void ack_flush(void *_completed)
167{
168}
169
170static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
171{
172 int i, cpu, me;
173 cpumask_var_t cpus;
174 bool called = true;
175 struct kvm_vcpu *vcpu;
176
177 zalloc_cpumask_var(&cpus, GFP_ATOMIC);
178
179 me = get_cpu();
180 kvm_for_each_vcpu(i, vcpu, kvm) {
181 kvm_make_request(req, vcpu);
182 cpu = vcpu->cpu;
183
184 /* Set ->requests bit before we read ->mode */
185 smp_mb();
186
187 if (cpus != NULL && cpu != -1 && cpu != me &&
188 kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
189 cpumask_set_cpu(cpu, cpus);
190 }
191 if (unlikely(cpus == NULL))
192 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
193 else if (!cpumask_empty(cpus))
194 smp_call_function_many(cpus, ack_flush, NULL, 1);
195 else
196 called = false;
197 put_cpu();
198 free_cpumask_var(cpus);
199 return called;
200}
201
202void kvm_flush_remote_tlbs(struct kvm *kvm)
203{
204 int dirty_count = kvm->tlbs_dirty;
205
206 smp_mb();
207 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
208 ++kvm->stat.remote_tlb_flush;
209 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
210}
211
212void kvm_reload_remote_mmus(struct kvm *kvm)
213{
214 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
215}
216
217int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
218{
219 struct page *page;
220 int r;
221
222 mutex_init(&vcpu->mutex);
223 vcpu->cpu = -1;
224 vcpu->kvm = kvm;
225 vcpu->vcpu_id = id;
226 vcpu->pid = NULL;
227 init_waitqueue_head(&vcpu->wq);
228 kvm_async_pf_vcpu_init(vcpu);
229
230 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
231 if (!page) {
232 r = -ENOMEM;
233 goto fail;
234 }
235 vcpu->run = page_address(page);
236
237 r = kvm_arch_vcpu_init(vcpu);
238 if (r < 0)
239 goto fail_free_run;
240 return 0;
241
242fail_free_run:
243 free_page((unsigned long)vcpu->run);
244fail:
245 return r;
246}
247EXPORT_SYMBOL_GPL(kvm_vcpu_init);
248
249void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
250{
251 put_pid(vcpu->pid);
252 kvm_arch_vcpu_uninit(vcpu);
253 free_page((unsigned long)vcpu->run);
254}
255EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
256
257#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
258static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
259{
260 return container_of(mn, struct kvm, mmu_notifier);
261}
262
263static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
264 struct mm_struct *mm,
265 unsigned long address)
266{
267 struct kvm *kvm = mmu_notifier_to_kvm(mn);
268 int need_tlb_flush, idx;
269
270 /*
271 * When ->invalidate_page runs, the linux pte has been zapped
272 * already but the page is still allocated until
273 * ->invalidate_page returns. So if we increase the sequence
274 * here the kvm page fault will notice if the spte can't be
275 * established because the page is going to be freed. If
276 * instead the kvm page fault establishes the spte before
277 * ->invalidate_page runs, kvm_unmap_hva will release it
278 * before returning.
279 *
280 * The sequence increase only need to be seen at spin_unlock
281 * time, and not at spin_lock time.
282 *
283 * Increasing the sequence after the spin_unlock would be
284 * unsafe because the kvm page fault could then establish the
285 * pte after kvm_unmap_hva returned, without noticing the page
286 * is going to be freed.
287 */
288 idx = srcu_read_lock(&kvm->srcu);
289 spin_lock(&kvm->mmu_lock);
290 kvm->mmu_notifier_seq++;
291 need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
292 spin_unlock(&kvm->mmu_lock);
293 srcu_read_unlock(&kvm->srcu, idx);
294
295 /* we've to flush the tlb before the pages can be freed */
296 if (need_tlb_flush)
297 kvm_flush_remote_tlbs(kvm);
298
299}
300
301static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
302 struct mm_struct *mm,
303 unsigned long address,
304 pte_t pte)
305{
306 struct kvm *kvm = mmu_notifier_to_kvm(mn);
307 int idx;
308
309 idx = srcu_read_lock(&kvm->srcu);
310 spin_lock(&kvm->mmu_lock);
311 kvm->mmu_notifier_seq++;
312 kvm_set_spte_hva(kvm, address, pte);
313 spin_unlock(&kvm->mmu_lock);
314 srcu_read_unlock(&kvm->srcu, idx);
315}
316
317static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
318 struct mm_struct *mm,
319 unsigned long start,
320 unsigned long end)
321{
322 struct kvm *kvm = mmu_notifier_to_kvm(mn);
323 int need_tlb_flush = 0, idx;
324
325 idx = srcu_read_lock(&kvm->srcu);
326 spin_lock(&kvm->mmu_lock);
327 /*
328 * The count increase must become visible at unlock time as no
329 * spte can be established without taking the mmu_lock and
330 * count is also read inside the mmu_lock critical section.
331 */
332 kvm->mmu_notifier_count++;
333 for (; start < end; start += PAGE_SIZE)
334 need_tlb_flush |= kvm_unmap_hva(kvm, start);
335 need_tlb_flush |= kvm->tlbs_dirty;
336 spin_unlock(&kvm->mmu_lock);
337 srcu_read_unlock(&kvm->srcu, idx);
338
339 /* we've to flush the tlb before the pages can be freed */
340 if (need_tlb_flush)
341 kvm_flush_remote_tlbs(kvm);
342}
343
344static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
345 struct mm_struct *mm,
346 unsigned long start,
347 unsigned long end)
348{
349 struct kvm *kvm = mmu_notifier_to_kvm(mn);
350
351 spin_lock(&kvm->mmu_lock);
352 /*
353 * This sequence increase will notify the kvm page fault that
354 * the page that is going to be mapped in the spte could have
355 * been freed.
356 */
357 kvm->mmu_notifier_seq++;
358 /*
359 * The above sequence increase must be visible before the
360 * below count decrease but both values are read by the kvm
361 * page fault under mmu_lock spinlock so we don't need to add
362 * a smb_wmb() here in between the two.
363 */
364 kvm->mmu_notifier_count--;
365 spin_unlock(&kvm->mmu_lock);
366
367 BUG_ON(kvm->mmu_notifier_count < 0);
368}
369
370static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
371 struct mm_struct *mm,
372 unsigned long address)
373{
374 struct kvm *kvm = mmu_notifier_to_kvm(mn);
375 int young, idx;
376
377 idx = srcu_read_lock(&kvm->srcu);
378 spin_lock(&kvm->mmu_lock);
379 young = kvm_age_hva(kvm, address);
380 spin_unlock(&kvm->mmu_lock);
381 srcu_read_unlock(&kvm->srcu, idx);
382
383 if (young)
384 kvm_flush_remote_tlbs(kvm);
385
386 return young;
387}
388
389static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
390 struct mm_struct *mm,
391 unsigned long address)
392{
393 struct kvm *kvm = mmu_notifier_to_kvm(mn);
394 int young, idx;
395
396 idx = srcu_read_lock(&kvm->srcu);
397 spin_lock(&kvm->mmu_lock);
398 young = kvm_test_age_hva(kvm, address);
399 spin_unlock(&kvm->mmu_lock);
400 srcu_read_unlock(&kvm->srcu, idx);
401
402 return young;
403}
404
405static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
406 struct mm_struct *mm)
407{
408 struct kvm *kvm = mmu_notifier_to_kvm(mn);
409 int idx;
410
411 idx = srcu_read_lock(&kvm->srcu);
412 kvm_arch_flush_shadow(kvm);
413 srcu_read_unlock(&kvm->srcu, idx);
414}
415
416static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
417 .invalidate_page = kvm_mmu_notifier_invalidate_page,
418 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
419 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
420 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
421 .test_young = kvm_mmu_notifier_test_young,
422 .change_pte = kvm_mmu_notifier_change_pte,
423 .release = kvm_mmu_notifier_release,
424};
425
426static int kvm_init_mmu_notifier(struct kvm *kvm)
427{
428 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
429 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
430}
431
432#else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
433
434static int kvm_init_mmu_notifier(struct kvm *kvm)
435{
436 return 0;
437}
438
439#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
440
441static struct kvm *kvm_create_vm(void)
442{
443 int r, i;
444 struct kvm *kvm = kvm_arch_alloc_vm();
445
446 if (!kvm)
447 return ERR_PTR(-ENOMEM);
448
449 r = kvm_arch_init_vm(kvm);
450 if (r)
451 goto out_err_nodisable;
452
453 r = hardware_enable_all();
454 if (r)
455 goto out_err_nodisable;
456
457#ifdef CONFIG_HAVE_KVM_IRQCHIP
458 INIT_HLIST_HEAD(&kvm->mask_notifier_list);
459 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
460#endif
461
462 r = -ENOMEM;
463 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
464 if (!kvm->memslots)
465 goto out_err_nosrcu;
466 if (init_srcu_struct(&kvm->srcu))
467 goto out_err_nosrcu;
468 for (i = 0; i < KVM_NR_BUSES; i++) {
469 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
470 GFP_KERNEL);
471 if (!kvm->buses[i])
472 goto out_err;
473 }
474
475 spin_lock_init(&kvm->mmu_lock);
476 kvm->mm = current->mm;
477 atomic_inc(&kvm->mm->mm_count);
478 kvm_eventfd_init(kvm);
479 mutex_init(&kvm->lock);
480 mutex_init(&kvm->irq_lock);
481 mutex_init(&kvm->slots_lock);
482 atomic_set(&kvm->users_count, 1);
483
484 r = kvm_init_mmu_notifier(kvm);
485 if (r)
486 goto out_err;
487
488 raw_spin_lock(&kvm_lock);
489 list_add(&kvm->vm_list, &vm_list);
490 raw_spin_unlock(&kvm_lock);
491
492 return kvm;
493
494out_err:
495 cleanup_srcu_struct(&kvm->srcu);
496out_err_nosrcu:
497 hardware_disable_all();
498out_err_nodisable:
499 for (i = 0; i < KVM_NR_BUSES; i++)
500 kfree(kvm->buses[i]);
501 kfree(kvm->memslots);
502 kvm_arch_free_vm(kvm);
503 return ERR_PTR(r);
504}
505
506static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
507{
508 if (!memslot->dirty_bitmap)
509 return;
510
511 if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE)
512 vfree(memslot->dirty_bitmap_head);
513 else
514 kfree(memslot->dirty_bitmap_head);
515
516 memslot->dirty_bitmap = NULL;
517 memslot->dirty_bitmap_head = NULL;
518}
519
520/*
521 * Free any memory in @free but not in @dont.
522 */
523static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
524 struct kvm_memory_slot *dont)
525{
526 int i;
527
528 if (!dont || free->rmap != dont->rmap)
529 vfree(free->rmap);
530
531 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
532 kvm_destroy_dirty_bitmap(free);
533
534
535 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
536 if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
537 vfree(free->lpage_info[i]);
538 free->lpage_info[i] = NULL;
539 }
540 }
541
542 free->npages = 0;
543 free->rmap = NULL;
544}
545
546void kvm_free_physmem(struct kvm *kvm)
547{
548 int i;
549 struct kvm_memslots *slots = kvm->memslots;
550
551 for (i = 0; i < slots->nmemslots; ++i)
552 kvm_free_physmem_slot(&slots->memslots[i], NULL);
553
554 kfree(kvm->memslots);
555}
556
557static void kvm_destroy_vm(struct kvm *kvm)
558{
559 int i;
560 struct mm_struct *mm = kvm->mm;
561
562 kvm_arch_sync_events(kvm);
563 raw_spin_lock(&kvm_lock);
564 list_del(&kvm->vm_list);
565 raw_spin_unlock(&kvm_lock);
566 kvm_free_irq_routing(kvm);
567 for (i = 0; i < KVM_NR_BUSES; i++)
568 kvm_io_bus_destroy(kvm->buses[i]);
569 kvm_coalesced_mmio_free(kvm);
570#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
571 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
572#else
573 kvm_arch_flush_shadow(kvm);
574#endif
575 kvm_arch_destroy_vm(kvm);
576 kvm_free_physmem(kvm);
577 cleanup_srcu_struct(&kvm->srcu);
578 kvm_arch_free_vm(kvm);
579 hardware_disable_all();
580 mmdrop(mm);
581}
582
583void kvm_get_kvm(struct kvm *kvm)
584{
585 atomic_inc(&kvm->users_count);
586}
587EXPORT_SYMBOL_GPL(kvm_get_kvm);
588
589void kvm_put_kvm(struct kvm *kvm)
590{
591 if (atomic_dec_and_test(&kvm->users_count))
592 kvm_destroy_vm(kvm);
593}
594EXPORT_SYMBOL_GPL(kvm_put_kvm);
595
596
597static int kvm_vm_release(struct inode *inode, struct file *filp)
598{
599 struct kvm *kvm = filp->private_data;
600
601 kvm_irqfd_release(kvm);
602
603 kvm_put_kvm(kvm);
604 return 0;
605}
606
607#ifndef CONFIG_S390
608/*
609 * Allocation size is twice as large as the actual dirty bitmap size.
610 * This makes it possible to do double buffering: see x86's
611 * kvm_vm_ioctl_get_dirty_log().
612 */
613static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
614{
615 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
616
617 if (dirty_bytes > PAGE_SIZE)
618 memslot->dirty_bitmap = vzalloc(dirty_bytes);
619 else
620 memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL);
621
622 if (!memslot->dirty_bitmap)
623 return -ENOMEM;
624
625 memslot->dirty_bitmap_head = memslot->dirty_bitmap;
626 return 0;
627}
628#endif /* !CONFIG_S390 */
629
630/*
631 * Allocate some memory and give it an address in the guest physical address
632 * space.
633 *
634 * Discontiguous memory is allowed, mostly for framebuffers.
635 *
636 * Must be called holding mmap_sem for write.
637 */
638int __kvm_set_memory_region(struct kvm *kvm,
639 struct kvm_userspace_memory_region *mem,
640 int user_alloc)
641{
642 int r;
643 gfn_t base_gfn;
644 unsigned long npages;
645 unsigned long i;
646 struct kvm_memory_slot *memslot;
647 struct kvm_memory_slot old, new;
648 struct kvm_memslots *slots, *old_memslots;
649
650 r = -EINVAL;
651 /* General sanity checks */
652 if (mem->memory_size & (PAGE_SIZE - 1))
653 goto out;
654 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
655 goto out;
656 /* We can read the guest memory with __xxx_user() later on. */
657 if (user_alloc &&
658 ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
659 !access_ok(VERIFY_WRITE,
660 (void __user *)(unsigned long)mem->userspace_addr,
661 mem->memory_size)))
662 goto out;
663 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
664 goto out;
665 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
666 goto out;
667
668 memslot = &kvm->memslots->memslots[mem->slot];
669 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
670 npages = mem->memory_size >> PAGE_SHIFT;
671
672 r = -EINVAL;
673 if (npages > KVM_MEM_MAX_NR_PAGES)
674 goto out;
675
676 if (!npages)
677 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
678
679 new = old = *memslot;
680
681 new.id = mem->slot;
682 new.base_gfn = base_gfn;
683 new.npages = npages;
684 new.flags = mem->flags;
685
686 /* Disallow changing a memory slot's size. */
687 r = -EINVAL;
688 if (npages && old.npages && npages != old.npages)
689 goto out_free;
690
691 /* Check for overlaps */
692 r = -EEXIST;
693 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
694 struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
695
696 if (s == memslot || !s->npages)
697 continue;
698 if (!((base_gfn + npages <= s->base_gfn) ||
699 (base_gfn >= s->base_gfn + s->npages)))
700 goto out_free;
701 }
702
703 /* Free page dirty bitmap if unneeded */
704 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
705 new.dirty_bitmap = NULL;
706
707 r = -ENOMEM;
708
709 /* Allocate if a slot is being created */
710#ifndef CONFIG_S390
711 if (npages && !new.rmap) {
712 new.rmap = vzalloc(npages * sizeof(*new.rmap));
713
714 if (!new.rmap)
715 goto out_free;
716
717 new.user_alloc = user_alloc;
718 new.userspace_addr = mem->userspace_addr;
719 }
720 if (!npages)
721 goto skip_lpage;
722
723 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
724 unsigned long ugfn;
725 unsigned long j;
726 int lpages;
727 int level = i + 2;
728
729 /* Avoid unused variable warning if no large pages */
730 (void)level;
731
732 if (new.lpage_info[i])
733 continue;
734
735 lpages = 1 + ((base_gfn + npages - 1)
736 >> KVM_HPAGE_GFN_SHIFT(level));
737 lpages -= base_gfn >> KVM_HPAGE_GFN_SHIFT(level);
738
739 new.lpage_info[i] = vzalloc(lpages * sizeof(*new.lpage_info[i]));
740
741 if (!new.lpage_info[i])
742 goto out_free;
743
744 if (base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
745 new.lpage_info[i][0].write_count = 1;
746 if ((base_gfn+npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
747 new.lpage_info[i][lpages - 1].write_count = 1;
748 ugfn = new.userspace_addr >> PAGE_SHIFT;
749 /*
750 * If the gfn and userspace address are not aligned wrt each
751 * other, or if explicitly asked to, disable large page
752 * support for this slot
753 */
754 if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
755 !largepages_enabled)
756 for (j = 0; j < lpages; ++j)
757 new.lpage_info[i][j].write_count = 1;
758 }
759
760skip_lpage:
761
762 /* Allocate page dirty bitmap if needed */
763 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
764 if (kvm_create_dirty_bitmap(&new) < 0)
765 goto out_free;
766 /* destroy any largepage mappings for dirty tracking */
767 }
768#else /* not defined CONFIG_S390 */
769 new.user_alloc = user_alloc;
770 if (user_alloc)
771 new.userspace_addr = mem->userspace_addr;
772#endif /* not defined CONFIG_S390 */
773
774 if (!npages) {
775 r = -ENOMEM;
776 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
777 if (!slots)
778 goto out_free;
779 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
780 if (mem->slot >= slots->nmemslots)
781 slots->nmemslots = mem->slot + 1;
782 slots->generation++;
783 slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
784
785 old_memslots = kvm->memslots;
786 rcu_assign_pointer(kvm->memslots, slots);
787 synchronize_srcu_expedited(&kvm->srcu);
788 /* From this point no new shadow pages pointing to a deleted
789 * memslot will be created.
790 *
791 * validation of sp->gfn happens in:
792 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
793 * - kvm_is_visible_gfn (mmu_check_roots)
794 */
795 kvm_arch_flush_shadow(kvm);
796 kfree(old_memslots);
797 }
798
799 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
800 if (r)
801 goto out_free;
802
803 /* map the pages in iommu page table */
804 if (npages) {
805 r = kvm_iommu_map_pages(kvm, &new);
806 if (r)
807 goto out_free;
808 }
809
810 r = -ENOMEM;
811 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
812 if (!slots)
813 goto out_free;
814 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
815 if (mem->slot >= slots->nmemslots)
816 slots->nmemslots = mem->slot + 1;
817 slots->generation++;
818
819 /* actual memory is freed via old in kvm_free_physmem_slot below */
820 if (!npages) {
821 new.rmap = NULL;
822 new.dirty_bitmap = NULL;
823 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
824 new.lpage_info[i] = NULL;
825 }
826
827 slots->memslots[mem->slot] = new;
828 old_memslots = kvm->memslots;
829 rcu_assign_pointer(kvm->memslots, slots);
830 synchronize_srcu_expedited(&kvm->srcu);
831
832 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
833
834 /*
835 * If the new memory slot is created, we need to clear all
836 * mmio sptes.
837 */
838 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
839 kvm_arch_flush_shadow(kvm);
840
841 kvm_free_physmem_slot(&old, &new);
842 kfree(old_memslots);
843
844 return 0;
845
846out_free:
847 kvm_free_physmem_slot(&new, &old);
848out:
849 return r;
850
851}
852EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
853
854int kvm_set_memory_region(struct kvm *kvm,
855 struct kvm_userspace_memory_region *mem,
856 int user_alloc)
857{
858 int r;
859
860 mutex_lock(&kvm->slots_lock);
861 r = __kvm_set_memory_region(kvm, mem, user_alloc);
862 mutex_unlock(&kvm->slots_lock);
863 return r;
864}
865EXPORT_SYMBOL_GPL(kvm_set_memory_region);
866
867int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
868 struct
869 kvm_userspace_memory_region *mem,
870 int user_alloc)
871{
872 if (mem->slot >= KVM_MEMORY_SLOTS)
873 return -EINVAL;
874 return kvm_set_memory_region(kvm, mem, user_alloc);
875}
876
877int kvm_get_dirty_log(struct kvm *kvm,
878 struct kvm_dirty_log *log, int *is_dirty)
879{
880 struct kvm_memory_slot *memslot;
881 int r, i;
882 unsigned long n;
883 unsigned long any = 0;
884
885 r = -EINVAL;
886 if (log->slot >= KVM_MEMORY_SLOTS)
887 goto out;
888
889 memslot = &kvm->memslots->memslots[log->slot];
890 r = -ENOENT;
891 if (!memslot->dirty_bitmap)
892 goto out;
893
894 n = kvm_dirty_bitmap_bytes(memslot);
895
896 for (i = 0; !any && i < n/sizeof(long); ++i)
897 any = memslot->dirty_bitmap[i];
898
899 r = -EFAULT;
900 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
901 goto out;
902
903 if (any)
904 *is_dirty = 1;
905
906 r = 0;
907out:
908 return r;
909}
910
911void kvm_disable_largepages(void)
912{
913 largepages_enabled = false;
914}
915EXPORT_SYMBOL_GPL(kvm_disable_largepages);
916
917int is_error_page(struct page *page)
918{
919 return page == bad_page || page == hwpoison_page || page == fault_page;
920}
921EXPORT_SYMBOL_GPL(is_error_page);
922
923int is_error_pfn(pfn_t pfn)
924{
925 return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
926}
927EXPORT_SYMBOL_GPL(is_error_pfn);
928
929int is_hwpoison_pfn(pfn_t pfn)
930{
931 return pfn == hwpoison_pfn;
932}
933EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
934
935int is_fault_pfn(pfn_t pfn)
936{
937 return pfn == fault_pfn;
938}
939EXPORT_SYMBOL_GPL(is_fault_pfn);
940
941int is_noslot_pfn(pfn_t pfn)
942{
943 return pfn == bad_pfn;
944}
945EXPORT_SYMBOL_GPL(is_noslot_pfn);
946
947int is_invalid_pfn(pfn_t pfn)
948{
949 return pfn == hwpoison_pfn || pfn == fault_pfn;
950}
951EXPORT_SYMBOL_GPL(is_invalid_pfn);
952
953static inline unsigned long bad_hva(void)
954{
955 return PAGE_OFFSET;
956}
957
958int kvm_is_error_hva(unsigned long addr)
959{
960 return addr == bad_hva();
961}
962EXPORT_SYMBOL_GPL(kvm_is_error_hva);
963
964static struct kvm_memory_slot *__gfn_to_memslot(struct kvm_memslots *slots,
965 gfn_t gfn)
966{
967 int i;
968
969 for (i = 0; i < slots->nmemslots; ++i) {
970 struct kvm_memory_slot *memslot = &slots->memslots[i];
971
972 if (gfn >= memslot->base_gfn
973 && gfn < memslot->base_gfn + memslot->npages)
974 return memslot;
975 }
976 return NULL;
977}
978
979struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
980{
981 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
982}
983EXPORT_SYMBOL_GPL(gfn_to_memslot);
984
985int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
986{
987 int i;
988 struct kvm_memslots *slots = kvm_memslots(kvm);
989
990 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
991 struct kvm_memory_slot *memslot = &slots->memslots[i];
992
993 if (memslot->flags & KVM_MEMSLOT_INVALID)
994 continue;
995
996 if (gfn >= memslot->base_gfn
997 && gfn < memslot->base_gfn + memslot->npages)
998 return 1;
999 }
1000 return 0;
1001}
1002EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1003
1004unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1005{
1006 struct vm_area_struct *vma;
1007 unsigned long addr, size;
1008
1009 size = PAGE_SIZE;
1010
1011 addr = gfn_to_hva(kvm, gfn);
1012 if (kvm_is_error_hva(addr))
1013 return PAGE_SIZE;
1014
1015 down_read(¤t->mm->mmap_sem);
1016 vma = find_vma(current->mm, addr);
1017 if (!vma)
1018 goto out;
1019
1020 size = vma_kernel_pagesize(vma);
1021
1022out:
1023 up_read(¤t->mm->mmap_sem);
1024
1025 return size;
1026}
1027
1028static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1029 gfn_t *nr_pages)
1030{
1031 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1032 return bad_hva();
1033
1034 if (nr_pages)
1035 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1036
1037 return gfn_to_hva_memslot(slot, gfn);
1038}
1039
1040unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1041{
1042 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1043}
1044EXPORT_SYMBOL_GPL(gfn_to_hva);
1045
1046static pfn_t get_fault_pfn(void)
1047{
1048 get_page(fault_page);
1049 return fault_pfn;
1050}
1051
1052int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1053 unsigned long start, int write, struct page **page)
1054{
1055 int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1056
1057 if (write)
1058 flags |= FOLL_WRITE;
1059
1060 return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1061}
1062
1063static inline int check_user_page_hwpoison(unsigned long addr)
1064{
1065 int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1066
1067 rc = __get_user_pages(current, current->mm, addr, 1,
1068 flags, NULL, NULL, NULL);
1069 return rc == -EHWPOISON;
1070}
1071
1072static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
1073 bool *async, bool write_fault, bool *writable)
1074{
1075 struct page *page[1];
1076 int npages = 0;
1077 pfn_t pfn;
1078
1079 /* we can do it either atomically or asynchronously, not both */
1080 BUG_ON(atomic && async);
1081
1082 BUG_ON(!write_fault && !writable);
1083
1084 if (writable)
1085 *writable = true;
1086
1087 if (atomic || async)
1088 npages = __get_user_pages_fast(addr, 1, 1, page);
1089
1090 if (unlikely(npages != 1) && !atomic) {
1091 might_sleep();
1092
1093 if (writable)
1094 *writable = write_fault;
1095
1096 if (async) {
1097 down_read(¤t->mm->mmap_sem);
1098 npages = get_user_page_nowait(current, current->mm,
1099 addr, write_fault, page);
1100 up_read(¤t->mm->mmap_sem);
1101 } else
1102 npages = get_user_pages_fast(addr, 1, write_fault,
1103 page);
1104
1105 /* map read fault as writable if possible */
1106 if (unlikely(!write_fault) && npages == 1) {
1107 struct page *wpage[1];
1108
1109 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1110 if (npages == 1) {
1111 *writable = true;
1112 put_page(page[0]);
1113 page[0] = wpage[0];
1114 }
1115 npages = 1;
1116 }
1117 }
1118
1119 if (unlikely(npages != 1)) {
1120 struct vm_area_struct *vma;
1121
1122 if (atomic)
1123 return get_fault_pfn();
1124
1125 down_read(¤t->mm->mmap_sem);
1126 if (npages == -EHWPOISON ||
1127 (!async && check_user_page_hwpoison(addr))) {
1128 up_read(¤t->mm->mmap_sem);
1129 get_page(hwpoison_page);
1130 return page_to_pfn(hwpoison_page);
1131 }
1132
1133 vma = find_vma_intersection(current->mm, addr, addr+1);
1134
1135 if (vma == NULL)
1136 pfn = get_fault_pfn();
1137 else if ((vma->vm_flags & VM_PFNMAP)) {
1138 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1139 vma->vm_pgoff;
1140 BUG_ON(!kvm_is_mmio_pfn(pfn));
1141 } else {
1142 if (async && (vma->vm_flags & VM_WRITE))
1143 *async = true;
1144 pfn = get_fault_pfn();
1145 }
1146 up_read(¤t->mm->mmap_sem);
1147 } else
1148 pfn = page_to_pfn(page[0]);
1149
1150 return pfn;
1151}
1152
1153pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
1154{
1155 return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
1156}
1157EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);
1158
1159static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1160 bool write_fault, bool *writable)
1161{
1162 unsigned long addr;
1163
1164 if (async)
1165 *async = false;
1166
1167 addr = gfn_to_hva(kvm, gfn);
1168 if (kvm_is_error_hva(addr)) {
1169 get_page(bad_page);
1170 return page_to_pfn(bad_page);
1171 }
1172
1173 return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
1174}
1175
1176pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1177{
1178 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1179}
1180EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1181
1182pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1183 bool write_fault, bool *writable)
1184{
1185 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1186}
1187EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1188
1189pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1190{
1191 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1192}
1193EXPORT_SYMBOL_GPL(gfn_to_pfn);
1194
1195pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1196 bool *writable)
1197{
1198 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1199}
1200EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1201
1202pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
1203 struct kvm_memory_slot *slot, gfn_t gfn)
1204{
1205 unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1206 return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
1207}
1208
1209int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1210 int nr_pages)
1211{
1212 unsigned long addr;
1213 gfn_t entry;
1214
1215 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1216 if (kvm_is_error_hva(addr))
1217 return -1;
1218
1219 if (entry < nr_pages)
1220 return 0;
1221
1222 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1223}
1224EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1225
1226struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1227{
1228 pfn_t pfn;
1229
1230 pfn = gfn_to_pfn(kvm, gfn);
1231 if (!kvm_is_mmio_pfn(pfn))
1232 return pfn_to_page(pfn);
1233
1234 WARN_ON(kvm_is_mmio_pfn(pfn));
1235
1236 get_page(bad_page);
1237 return bad_page;
1238}
1239
1240EXPORT_SYMBOL_GPL(gfn_to_page);
1241
1242void kvm_release_page_clean(struct page *page)
1243{
1244 kvm_release_pfn_clean(page_to_pfn(page));
1245}
1246EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1247
1248void kvm_release_pfn_clean(pfn_t pfn)
1249{
1250 if (!kvm_is_mmio_pfn(pfn))
1251 put_page(pfn_to_page(pfn));
1252}
1253EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1254
1255void kvm_release_page_dirty(struct page *page)
1256{
1257 kvm_release_pfn_dirty(page_to_pfn(page));
1258}
1259EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1260
1261void kvm_release_pfn_dirty(pfn_t pfn)
1262{
1263 kvm_set_pfn_dirty(pfn);
1264 kvm_release_pfn_clean(pfn);
1265}
1266EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1267
1268void kvm_set_page_dirty(struct page *page)
1269{
1270 kvm_set_pfn_dirty(page_to_pfn(page));
1271}
1272EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1273
1274void kvm_set_pfn_dirty(pfn_t pfn)
1275{
1276 if (!kvm_is_mmio_pfn(pfn)) {
1277 struct page *page = pfn_to_page(pfn);
1278 if (!PageReserved(page))
1279 SetPageDirty(page);
1280 }
1281}
1282EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1283
1284void kvm_set_pfn_accessed(pfn_t pfn)
1285{
1286 if (!kvm_is_mmio_pfn(pfn))
1287 mark_page_accessed(pfn_to_page(pfn));
1288}
1289EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1290
1291void kvm_get_pfn(pfn_t pfn)
1292{
1293 if (!kvm_is_mmio_pfn(pfn))
1294 get_page(pfn_to_page(pfn));
1295}
1296EXPORT_SYMBOL_GPL(kvm_get_pfn);
1297
1298static int next_segment(unsigned long len, int offset)
1299{
1300 if (len > PAGE_SIZE - offset)
1301 return PAGE_SIZE - offset;
1302 else
1303 return len;
1304}
1305
1306int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1307 int len)
1308{
1309 int r;
1310 unsigned long addr;
1311
1312 addr = gfn_to_hva(kvm, gfn);
1313 if (kvm_is_error_hva(addr))
1314 return -EFAULT;
1315 r = __copy_from_user(data, (void __user *)addr + offset, len);
1316 if (r)
1317 return -EFAULT;
1318 return 0;
1319}
1320EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1321
1322int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1323{
1324 gfn_t gfn = gpa >> PAGE_SHIFT;
1325 int seg;
1326 int offset = offset_in_page(gpa);
1327 int ret;
1328
1329 while ((seg = next_segment(len, offset)) != 0) {
1330 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1331 if (ret < 0)
1332 return ret;
1333 offset = 0;
1334 len -= seg;
1335 data += seg;
1336 ++gfn;
1337 }
1338 return 0;
1339}
1340EXPORT_SYMBOL_GPL(kvm_read_guest);
1341
1342int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1343 unsigned long len)
1344{
1345 int r;
1346 unsigned long addr;
1347 gfn_t gfn = gpa >> PAGE_SHIFT;
1348 int offset = offset_in_page(gpa);
1349
1350 addr = gfn_to_hva(kvm, gfn);
1351 if (kvm_is_error_hva(addr))
1352 return -EFAULT;
1353 pagefault_disable();
1354 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1355 pagefault_enable();
1356 if (r)
1357 return -EFAULT;
1358 return 0;
1359}
1360EXPORT_SYMBOL(kvm_read_guest_atomic);
1361
1362int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1363 int offset, int len)
1364{
1365 int r;
1366 unsigned long addr;
1367
1368 addr = gfn_to_hva(kvm, gfn);
1369 if (kvm_is_error_hva(addr))
1370 return -EFAULT;
1371 r = __copy_to_user((void __user *)addr + offset, data, len);
1372 if (r)
1373 return -EFAULT;
1374 mark_page_dirty(kvm, gfn);
1375 return 0;
1376}
1377EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1378
1379int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1380 unsigned long len)
1381{
1382 gfn_t gfn = gpa >> PAGE_SHIFT;
1383 int seg;
1384 int offset = offset_in_page(gpa);
1385 int ret;
1386
1387 while ((seg = next_segment(len, offset)) != 0) {
1388 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1389 if (ret < 0)
1390 return ret;
1391 offset = 0;
1392 len -= seg;
1393 data += seg;
1394 ++gfn;
1395 }
1396 return 0;
1397}
1398
1399int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1400 gpa_t gpa)
1401{
1402 struct kvm_memslots *slots = kvm_memslots(kvm);
1403 int offset = offset_in_page(gpa);
1404 gfn_t gfn = gpa >> PAGE_SHIFT;
1405
1406 ghc->gpa = gpa;
1407 ghc->generation = slots->generation;
1408 ghc->memslot = __gfn_to_memslot(slots, gfn);
1409 ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
1410 if (!kvm_is_error_hva(ghc->hva))
1411 ghc->hva += offset;
1412 else
1413 return -EFAULT;
1414
1415 return 0;
1416}
1417EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1418
1419int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1420 void *data, unsigned long len)
1421{
1422 struct kvm_memslots *slots = kvm_memslots(kvm);
1423 int r;
1424
1425 if (slots->generation != ghc->generation)
1426 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1427
1428 if (kvm_is_error_hva(ghc->hva))
1429 return -EFAULT;
1430
1431 r = __copy_to_user((void __user *)ghc->hva, data, len);
1432 if (r)
1433 return -EFAULT;
1434 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1435
1436 return 0;
1437}
1438EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1439
1440int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1441 void *data, unsigned long len)
1442{
1443 struct kvm_memslots *slots = kvm_memslots(kvm);
1444 int r;
1445
1446 if (slots->generation != ghc->generation)
1447 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1448
1449 if (kvm_is_error_hva(ghc->hva))
1450 return -EFAULT;
1451
1452 r = __copy_from_user(data, (void __user *)ghc->hva, len);
1453 if (r)
1454 return -EFAULT;
1455
1456 return 0;
1457}
1458EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1459
1460int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1461{
1462 return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
1463 offset, len);
1464}
1465EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1466
1467int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1468{
1469 gfn_t gfn = gpa >> PAGE_SHIFT;
1470 int seg;
1471 int offset = offset_in_page(gpa);
1472 int ret;
1473
1474 while ((seg = next_segment(len, offset)) != 0) {
1475 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1476 if (ret < 0)
1477 return ret;
1478 offset = 0;
1479 len -= seg;
1480 ++gfn;
1481 }
1482 return 0;
1483}
1484EXPORT_SYMBOL_GPL(kvm_clear_guest);
1485
1486void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
1487 gfn_t gfn)
1488{
1489 if (memslot && memslot->dirty_bitmap) {
1490 unsigned long rel_gfn = gfn - memslot->base_gfn;
1491
1492 __set_bit_le(rel_gfn, memslot->dirty_bitmap);
1493 }
1494}
1495
1496void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1497{
1498 struct kvm_memory_slot *memslot;
1499
1500 memslot = gfn_to_memslot(kvm, gfn);
1501 mark_page_dirty_in_slot(kvm, memslot, gfn);
1502}
1503
1504/*
1505 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1506 */
1507void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1508{
1509 DEFINE_WAIT(wait);
1510
1511 for (;;) {
1512 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1513
1514 if (kvm_arch_vcpu_runnable(vcpu)) {
1515 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1516 break;
1517 }
1518 if (kvm_cpu_has_pending_timer(vcpu))
1519 break;
1520 if (signal_pending(current))
1521 break;
1522
1523 schedule();
1524 }
1525
1526 finish_wait(&vcpu->wq, &wait);
1527}
1528
1529void kvm_resched(struct kvm_vcpu *vcpu)
1530{
1531 if (!need_resched())
1532 return;
1533 cond_resched();
1534}
1535EXPORT_SYMBOL_GPL(kvm_resched);
1536
1537void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1538{
1539 struct kvm *kvm = me->kvm;
1540 struct kvm_vcpu *vcpu;
1541 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1542 int yielded = 0;
1543 int pass;
1544 int i;
1545
1546 /*
1547 * We boost the priority of a VCPU that is runnable but not
1548 * currently running, because it got preempted by something
1549 * else and called schedule in __vcpu_run. Hopefully that
1550 * VCPU is holding the lock that we need and will release it.
1551 * We approximate round-robin by starting at the last boosted VCPU.
1552 */
1553 for (pass = 0; pass < 2 && !yielded; pass++) {
1554 kvm_for_each_vcpu(i, vcpu, kvm) {
1555 struct task_struct *task = NULL;
1556 struct pid *pid;
1557 if (!pass && i < last_boosted_vcpu) {
1558 i = last_boosted_vcpu;
1559 continue;
1560 } else if (pass && i > last_boosted_vcpu)
1561 break;
1562 if (vcpu == me)
1563 continue;
1564 if (waitqueue_active(&vcpu->wq))
1565 continue;
1566 rcu_read_lock();
1567 pid = rcu_dereference(vcpu->pid);
1568 if (pid)
1569 task = get_pid_task(vcpu->pid, PIDTYPE_PID);
1570 rcu_read_unlock();
1571 if (!task)
1572 continue;
1573 if (task->flags & PF_VCPU) {
1574 put_task_struct(task);
1575 continue;
1576 }
1577 if (yield_to(task, 1)) {
1578 put_task_struct(task);
1579 kvm->last_boosted_vcpu = i;
1580 yielded = 1;
1581 break;
1582 }
1583 put_task_struct(task);
1584 }
1585 }
1586}
1587EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1588
1589static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1590{
1591 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1592 struct page *page;
1593
1594 if (vmf->pgoff == 0)
1595 page = virt_to_page(vcpu->run);
1596#ifdef CONFIG_X86
1597 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1598 page = virt_to_page(vcpu->arch.pio_data);
1599#endif
1600#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1601 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1602 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1603#endif
1604 else
1605 return VM_FAULT_SIGBUS;
1606 get_page(page);
1607 vmf->page = page;
1608 return 0;
1609}
1610
1611static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1612 .fault = kvm_vcpu_fault,
1613};
1614
1615static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1616{
1617 vma->vm_ops = &kvm_vcpu_vm_ops;
1618 return 0;
1619}
1620
1621static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1622{
1623 struct kvm_vcpu *vcpu = filp->private_data;
1624
1625 kvm_put_kvm(vcpu->kvm);
1626 return 0;
1627}
1628
1629static struct file_operations kvm_vcpu_fops = {
1630 .release = kvm_vcpu_release,
1631 .unlocked_ioctl = kvm_vcpu_ioctl,
1632#ifdef CONFIG_COMPAT
1633 .compat_ioctl = kvm_vcpu_compat_ioctl,
1634#endif
1635 .mmap = kvm_vcpu_mmap,
1636 .llseek = noop_llseek,
1637};
1638
1639/*
1640 * Allocates an inode for the vcpu.
1641 */
1642static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1643{
1644 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
1645}
1646
1647/*
1648 * Creates some virtual cpus. Good luck creating more than one.
1649 */
1650static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1651{
1652 int r;
1653 struct kvm_vcpu *vcpu, *v;
1654
1655 vcpu = kvm_arch_vcpu_create(kvm, id);
1656 if (IS_ERR(vcpu))
1657 return PTR_ERR(vcpu);
1658
1659 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1660
1661 r = kvm_arch_vcpu_setup(vcpu);
1662 if (r)
1663 goto vcpu_destroy;
1664
1665 mutex_lock(&kvm->lock);
1666 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1667 r = -EINVAL;
1668 goto unlock_vcpu_destroy;
1669 }
1670
1671 kvm_for_each_vcpu(r, v, kvm)
1672 if (v->vcpu_id == id) {
1673 r = -EEXIST;
1674 goto unlock_vcpu_destroy;
1675 }
1676
1677 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1678
1679 /* Now it's all set up, let userspace reach it */
1680 kvm_get_kvm(kvm);
1681 r = create_vcpu_fd(vcpu);
1682 if (r < 0) {
1683 kvm_put_kvm(kvm);
1684 goto unlock_vcpu_destroy;
1685 }
1686
1687 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1688 smp_wmb();
1689 atomic_inc(&kvm->online_vcpus);
1690
1691#ifdef CONFIG_KVM_APIC_ARCHITECTURE
1692 if (kvm->bsp_vcpu_id == id)
1693 kvm->bsp_vcpu = vcpu;
1694#endif
1695 mutex_unlock(&kvm->lock);
1696 return r;
1697
1698unlock_vcpu_destroy:
1699 mutex_unlock(&kvm->lock);
1700vcpu_destroy:
1701 kvm_arch_vcpu_destroy(vcpu);
1702 return r;
1703}
1704
1705static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1706{
1707 if (sigset) {
1708 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1709 vcpu->sigset_active = 1;
1710 vcpu->sigset = *sigset;
1711 } else
1712 vcpu->sigset_active = 0;
1713 return 0;
1714}
1715
1716static long kvm_vcpu_ioctl(struct file *filp,
1717 unsigned int ioctl, unsigned long arg)
1718{
1719 struct kvm_vcpu *vcpu = filp->private_data;
1720 void __user *argp = (void __user *)arg;
1721 int r;
1722 struct kvm_fpu *fpu = NULL;
1723 struct kvm_sregs *kvm_sregs = NULL;
1724
1725 if (vcpu->kvm->mm != current->mm)
1726 return -EIO;
1727
1728#if defined(CONFIG_S390) || defined(CONFIG_PPC)
1729 /*
1730 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1731 * so vcpu_load() would break it.
1732 */
1733 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1734 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1735#endif
1736
1737
1738 vcpu_load(vcpu);
1739 switch (ioctl) {
1740 case KVM_RUN:
1741 r = -EINVAL;
1742 if (arg)
1743 goto out;
1744 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1745 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
1746 break;
1747 case KVM_GET_REGS: {
1748 struct kvm_regs *kvm_regs;
1749
1750 r = -ENOMEM;
1751 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1752 if (!kvm_regs)
1753 goto out;
1754 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1755 if (r)
1756 goto out_free1;
1757 r = -EFAULT;
1758 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1759 goto out_free1;
1760 r = 0;
1761out_free1:
1762 kfree(kvm_regs);
1763 break;
1764 }
1765 case KVM_SET_REGS: {
1766 struct kvm_regs *kvm_regs;
1767
1768 r = -ENOMEM;
1769 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1770 if (!kvm_regs)
1771 goto out;
1772 r = -EFAULT;
1773 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1774 goto out_free2;
1775 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1776 if (r)
1777 goto out_free2;
1778 r = 0;
1779out_free2:
1780 kfree(kvm_regs);
1781 break;
1782 }
1783 case KVM_GET_SREGS: {
1784 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1785 r = -ENOMEM;
1786 if (!kvm_sregs)
1787 goto out;
1788 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
1789 if (r)
1790 goto out;
1791 r = -EFAULT;
1792 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
1793 goto out;
1794 r = 0;
1795 break;
1796 }
1797 case KVM_SET_SREGS: {
1798 kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1799 r = -ENOMEM;
1800 if (!kvm_sregs)
1801 goto out;
1802 r = -EFAULT;
1803 if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
1804 goto out;
1805 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
1806 if (r)
1807 goto out;
1808 r = 0;
1809 break;
1810 }
1811 case KVM_GET_MP_STATE: {
1812 struct kvm_mp_state mp_state;
1813
1814 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1815 if (r)
1816 goto out;
1817 r = -EFAULT;
1818 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1819 goto out;
1820 r = 0;
1821 break;
1822 }
1823 case KVM_SET_MP_STATE: {
1824 struct kvm_mp_state mp_state;
1825
1826 r = -EFAULT;
1827 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1828 goto out;
1829 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1830 if (r)
1831 goto out;
1832 r = 0;
1833 break;
1834 }
1835 case KVM_TRANSLATE: {
1836 struct kvm_translation tr;
1837
1838 r = -EFAULT;
1839 if (copy_from_user(&tr, argp, sizeof tr))
1840 goto out;
1841 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1842 if (r)
1843 goto out;
1844 r = -EFAULT;
1845 if (copy_to_user(argp, &tr, sizeof tr))
1846 goto out;
1847 r = 0;
1848 break;
1849 }
1850 case KVM_SET_GUEST_DEBUG: {
1851 struct kvm_guest_debug dbg;
1852
1853 r = -EFAULT;
1854 if (copy_from_user(&dbg, argp, sizeof dbg))
1855 goto out;
1856 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
1857 if (r)
1858 goto out;
1859 r = 0;
1860 break;
1861 }
1862 case KVM_SET_SIGNAL_MASK: {
1863 struct kvm_signal_mask __user *sigmask_arg = argp;
1864 struct kvm_signal_mask kvm_sigmask;
1865 sigset_t sigset, *p;
1866
1867 p = NULL;
1868 if (argp) {
1869 r = -EFAULT;
1870 if (copy_from_user(&kvm_sigmask, argp,
1871 sizeof kvm_sigmask))
1872 goto out;
1873 r = -EINVAL;
1874 if (kvm_sigmask.len != sizeof sigset)
1875 goto out;
1876 r = -EFAULT;
1877 if (copy_from_user(&sigset, sigmask_arg->sigset,
1878 sizeof sigset))
1879 goto out;
1880 p = &sigset;
1881 }
1882 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
1883 break;
1884 }
1885 case KVM_GET_FPU: {
1886 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1887 r = -ENOMEM;
1888 if (!fpu)
1889 goto out;
1890 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
1891 if (r)
1892 goto out;
1893 r = -EFAULT;
1894 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
1895 goto out;
1896 r = 0;
1897 break;
1898 }
1899 case KVM_SET_FPU: {
1900 fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1901 r = -ENOMEM;
1902 if (!fpu)
1903 goto out;
1904 r = -EFAULT;
1905 if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
1906 goto out;
1907 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
1908 if (r)
1909 goto out;
1910 r = 0;
1911 break;
1912 }
1913 default:
1914 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1915 }
1916out:
1917 vcpu_put(vcpu);
1918 kfree(fpu);
1919 kfree(kvm_sregs);
1920 return r;
1921}
1922
1923#ifdef CONFIG_COMPAT
1924static long kvm_vcpu_compat_ioctl(struct file *filp,
1925 unsigned int ioctl, unsigned long arg)
1926{
1927 struct kvm_vcpu *vcpu = filp->private_data;
1928 void __user *argp = compat_ptr(arg);
1929 int r;
1930
1931 if (vcpu->kvm->mm != current->mm)
1932 return -EIO;
1933
1934 switch (ioctl) {
1935 case KVM_SET_SIGNAL_MASK: {
1936 struct kvm_signal_mask __user *sigmask_arg = argp;
1937 struct kvm_signal_mask kvm_sigmask;
1938 compat_sigset_t csigset;
1939 sigset_t sigset;
1940
1941 if (argp) {
1942 r = -EFAULT;
1943 if (copy_from_user(&kvm_sigmask, argp,
1944 sizeof kvm_sigmask))
1945 goto out;
1946 r = -EINVAL;
1947 if (kvm_sigmask.len != sizeof csigset)
1948 goto out;
1949 r = -EFAULT;
1950 if (copy_from_user(&csigset, sigmask_arg->sigset,
1951 sizeof csigset))
1952 goto out;
1953 }
1954 sigset_from_compat(&sigset, &csigset);
1955 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1956 break;
1957 }
1958 default:
1959 r = kvm_vcpu_ioctl(filp, ioctl, arg);
1960 }
1961
1962out:
1963 return r;
1964}
1965#endif
1966
1967static long kvm_vm_ioctl(struct file *filp,
1968 unsigned int ioctl, unsigned long arg)
1969{
1970 struct kvm *kvm = filp->private_data;
1971 void __user *argp = (void __user *)arg;
1972 int r;
1973
1974 if (kvm->mm != current->mm)
1975 return -EIO;
1976 switch (ioctl) {
1977 case KVM_CREATE_VCPU:
1978 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1979 if (r < 0)
1980 goto out;
1981 break;
1982 case KVM_SET_USER_MEMORY_REGION: {
1983 struct kvm_userspace_memory_region kvm_userspace_mem;
1984
1985 r = -EFAULT;
1986 if (copy_from_user(&kvm_userspace_mem, argp,
1987 sizeof kvm_userspace_mem))
1988 goto out;
1989
1990 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1991 if (r)
1992 goto out;
1993 break;
1994 }
1995 case KVM_GET_DIRTY_LOG: {
1996 struct kvm_dirty_log log;
1997
1998 r = -EFAULT;
1999 if (copy_from_user(&log, argp, sizeof log))
2000 goto out;
2001 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2002 if (r)
2003 goto out;
2004 break;
2005 }
2006#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2007 case KVM_REGISTER_COALESCED_MMIO: {
2008 struct kvm_coalesced_mmio_zone zone;
2009 r = -EFAULT;
2010 if (copy_from_user(&zone, argp, sizeof zone))
2011 goto out;
2012 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2013 if (r)
2014 goto out;
2015 r = 0;
2016 break;
2017 }
2018 case KVM_UNREGISTER_COALESCED_MMIO: {
2019 struct kvm_coalesced_mmio_zone zone;
2020 r = -EFAULT;
2021 if (copy_from_user(&zone, argp, sizeof zone))
2022 goto out;
2023 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2024 if (r)
2025 goto out;
2026 r = 0;
2027 break;
2028 }
2029#endif
2030 case KVM_IRQFD: {
2031 struct kvm_irqfd data;
2032
2033 r = -EFAULT;
2034 if (copy_from_user(&data, argp, sizeof data))
2035 goto out;
2036 r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
2037 break;
2038 }
2039 case KVM_IOEVENTFD: {
2040 struct kvm_ioeventfd data;
2041
2042 r = -EFAULT;
2043 if (copy_from_user(&data, argp, sizeof data))
2044 goto out;
2045 r = kvm_ioeventfd(kvm, &data);
2046 break;
2047 }
2048#ifdef CONFIG_KVM_APIC_ARCHITECTURE
2049 case KVM_SET_BOOT_CPU_ID:
2050 r = 0;
2051 mutex_lock(&kvm->lock);
2052 if (atomic_read(&kvm->online_vcpus) != 0)
2053 r = -EBUSY;
2054 else
2055 kvm->bsp_vcpu_id = arg;
2056 mutex_unlock(&kvm->lock);
2057 break;
2058#endif
2059 default:
2060 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2061 if (r == -ENOTTY)
2062 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2063 }
2064out:
2065 return r;
2066}
2067
2068#ifdef CONFIG_COMPAT
2069struct compat_kvm_dirty_log {
2070 __u32 slot;
2071 __u32 padding1;
2072 union {
2073 compat_uptr_t dirty_bitmap; /* one bit per page */
2074 __u64 padding2;
2075 };
2076};
2077
2078static long kvm_vm_compat_ioctl(struct file *filp,
2079 unsigned int ioctl, unsigned long arg)
2080{
2081 struct kvm *kvm = filp->private_data;
2082 int r;
2083
2084 if (kvm->mm != current->mm)
2085 return -EIO;
2086 switch (ioctl) {
2087 case KVM_GET_DIRTY_LOG: {
2088 struct compat_kvm_dirty_log compat_log;
2089 struct kvm_dirty_log log;
2090
2091 r = -EFAULT;
2092 if (copy_from_user(&compat_log, (void __user *)arg,
2093 sizeof(compat_log)))
2094 goto out;
2095 log.slot = compat_log.slot;
2096 log.padding1 = compat_log.padding1;
2097 log.padding2 = compat_log.padding2;
2098 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2099
2100 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2101 if (r)
2102 goto out;
2103 break;
2104 }
2105 default:
2106 r = kvm_vm_ioctl(filp, ioctl, arg);
2107 }
2108
2109out:
2110 return r;
2111}
2112#endif
2113
2114static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2115{
2116 struct page *page[1];
2117 unsigned long addr;
2118 int npages;
2119 gfn_t gfn = vmf->pgoff;
2120 struct kvm *kvm = vma->vm_file->private_data;
2121
2122 addr = gfn_to_hva(kvm, gfn);
2123 if (kvm_is_error_hva(addr))
2124 return VM_FAULT_SIGBUS;
2125
2126 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
2127 NULL);
2128 if (unlikely(npages != 1))
2129 return VM_FAULT_SIGBUS;
2130
2131 vmf->page = page[0];
2132 return 0;
2133}
2134
2135static const struct vm_operations_struct kvm_vm_vm_ops = {
2136 .fault = kvm_vm_fault,
2137};
2138
2139static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2140{
2141 vma->vm_ops = &kvm_vm_vm_ops;
2142 return 0;
2143}
2144
2145static struct file_operations kvm_vm_fops = {
2146 .release = kvm_vm_release,
2147 .unlocked_ioctl = kvm_vm_ioctl,
2148#ifdef CONFIG_COMPAT
2149 .compat_ioctl = kvm_vm_compat_ioctl,
2150#endif
2151 .mmap = kvm_vm_mmap,
2152 .llseek = noop_llseek,
2153};
2154
2155static int kvm_dev_ioctl_create_vm(void)
2156{
2157 int r;
2158 struct kvm *kvm;
2159
2160 kvm = kvm_create_vm();
2161 if (IS_ERR(kvm))
2162 return PTR_ERR(kvm);
2163#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2164 r = kvm_coalesced_mmio_init(kvm);
2165 if (r < 0) {
2166 kvm_put_kvm(kvm);
2167 return r;
2168 }
2169#endif
2170 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
2171 if (r < 0)
2172 kvm_put_kvm(kvm);
2173
2174 return r;
2175}
2176
2177static long kvm_dev_ioctl_check_extension_generic(long arg)
2178{
2179 switch (arg) {
2180 case KVM_CAP_USER_MEMORY:
2181 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2182 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2183#ifdef CONFIG_KVM_APIC_ARCHITECTURE
2184 case KVM_CAP_SET_BOOT_CPU_ID:
2185#endif
2186 case KVM_CAP_INTERNAL_ERROR_DATA:
2187 return 1;
2188#ifdef CONFIG_HAVE_KVM_IRQCHIP
2189 case KVM_CAP_IRQ_ROUTING:
2190 return KVM_MAX_IRQ_ROUTES;
2191#endif
2192 default:
2193 break;
2194 }
2195 return kvm_dev_ioctl_check_extension(arg);
2196}
2197
2198static long kvm_dev_ioctl(struct file *filp,
2199 unsigned int ioctl, unsigned long arg)
2200{
2201 long r = -EINVAL;
2202
2203 switch (ioctl) {
2204 case KVM_GET_API_VERSION:
2205 r = -EINVAL;
2206 if (arg)
2207 goto out;
2208 r = KVM_API_VERSION;
2209 break;
2210 case KVM_CREATE_VM:
2211 r = -EINVAL;
2212 if (arg)
2213 goto out;
2214 r = kvm_dev_ioctl_create_vm();
2215 break;
2216 case KVM_CHECK_EXTENSION:
2217 r = kvm_dev_ioctl_check_extension_generic(arg);
2218 break;
2219 case KVM_GET_VCPU_MMAP_SIZE:
2220 r = -EINVAL;
2221 if (arg)
2222 goto out;
2223 r = PAGE_SIZE; /* struct kvm_run */
2224#ifdef CONFIG_X86
2225 r += PAGE_SIZE; /* pio data page */
2226#endif
2227#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2228 r += PAGE_SIZE; /* coalesced mmio ring page */
2229#endif
2230 break;
2231 case KVM_TRACE_ENABLE:
2232 case KVM_TRACE_PAUSE:
2233 case KVM_TRACE_DISABLE:
2234 r = -EOPNOTSUPP;
2235 break;
2236 default:
2237 return kvm_arch_dev_ioctl(filp, ioctl, arg);
2238 }
2239out:
2240 return r;
2241}
2242
2243static struct file_operations kvm_chardev_ops = {
2244 .unlocked_ioctl = kvm_dev_ioctl,
2245 .compat_ioctl = kvm_dev_ioctl,
2246 .llseek = noop_llseek,
2247};
2248
2249static struct miscdevice kvm_dev = {
2250 KVM_MINOR,
2251 "kvm",
2252 &kvm_chardev_ops,
2253};
2254
2255static void hardware_enable_nolock(void *junk)
2256{
2257 int cpu = raw_smp_processor_id();
2258 int r;
2259
2260 if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2261 return;
2262
2263 cpumask_set_cpu(cpu, cpus_hardware_enabled);
2264
2265 r = kvm_arch_hardware_enable(NULL);
2266
2267 if (r) {
2268 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2269 atomic_inc(&hardware_enable_failed);
2270 printk(KERN_INFO "kvm: enabling virtualization on "
2271 "CPU%d failed\n", cpu);
2272 }
2273}
2274
2275static void hardware_enable(void *junk)
2276{
2277 raw_spin_lock(&kvm_lock);
2278 hardware_enable_nolock(junk);
2279 raw_spin_unlock(&kvm_lock);
2280}
2281
2282static void hardware_disable_nolock(void *junk)
2283{
2284 int cpu = raw_smp_processor_id();
2285
2286 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2287 return;
2288 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2289 kvm_arch_hardware_disable(NULL);
2290}
2291
2292static void hardware_disable(void *junk)
2293{
2294 raw_spin_lock(&kvm_lock);
2295 hardware_disable_nolock(junk);
2296 raw_spin_unlock(&kvm_lock);
2297}
2298
2299static void hardware_disable_all_nolock(void)
2300{
2301 BUG_ON(!kvm_usage_count);
2302
2303 kvm_usage_count--;
2304 if (!kvm_usage_count)
2305 on_each_cpu(hardware_disable_nolock, NULL, 1);
2306}
2307
2308static void hardware_disable_all(void)
2309{
2310 raw_spin_lock(&kvm_lock);
2311 hardware_disable_all_nolock();
2312 raw_spin_unlock(&kvm_lock);
2313}
2314
2315static int hardware_enable_all(void)
2316{
2317 int r = 0;
2318
2319 raw_spin_lock(&kvm_lock);
2320
2321 kvm_usage_count++;
2322 if (kvm_usage_count == 1) {
2323 atomic_set(&hardware_enable_failed, 0);
2324 on_each_cpu(hardware_enable_nolock, NULL, 1);
2325
2326 if (atomic_read(&hardware_enable_failed)) {
2327 hardware_disable_all_nolock();
2328 r = -EBUSY;
2329 }
2330 }
2331
2332 raw_spin_unlock(&kvm_lock);
2333
2334 return r;
2335}
2336
2337static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2338 void *v)
2339{
2340 int cpu = (long)v;
2341
2342 if (!kvm_usage_count)
2343 return NOTIFY_OK;
2344
2345 val &= ~CPU_TASKS_FROZEN;
2346 switch (val) {
2347 case CPU_DYING:
2348 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2349 cpu);
2350 hardware_disable(NULL);
2351 break;
2352 case CPU_STARTING:
2353 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2354 cpu);
2355 hardware_enable(NULL);
2356 break;
2357 }
2358 return NOTIFY_OK;
2359}
2360
2361
2362asmlinkage void kvm_spurious_fault(void)
2363{
2364 /* Fault while not rebooting. We want the trace. */
2365 BUG();
2366}
2367EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2368
2369static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2370 void *v)
2371{
2372 /*
2373 * Some (well, at least mine) BIOSes hang on reboot if
2374 * in vmx root mode.
2375 *
2376 * And Intel TXT required VMX off for all cpu when system shutdown.
2377 */
2378 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2379 kvm_rebooting = true;
2380 on_each_cpu(hardware_disable_nolock, NULL, 1);
2381 return NOTIFY_OK;
2382}
2383
2384static struct notifier_block kvm_reboot_notifier = {
2385 .notifier_call = kvm_reboot,
2386 .priority = 0,
2387};
2388
2389static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2390{
2391 int i;
2392
2393 for (i = 0; i < bus->dev_count; i++) {
2394 struct kvm_io_device *pos = bus->devs[i];
2395
2396 kvm_iodevice_destructor(pos);
2397 }
2398 kfree(bus);
2399}
2400
2401/* kvm_io_bus_write - called under kvm->slots_lock */
2402int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2403 int len, const void *val)
2404{
2405 int i;
2406 struct kvm_io_bus *bus;
2407
2408 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2409 for (i = 0; i < bus->dev_count; i++)
2410 if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
2411 return 0;
2412 return -EOPNOTSUPP;
2413}
2414
2415/* kvm_io_bus_read - called under kvm->slots_lock */
2416int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2417 int len, void *val)
2418{
2419 int i;
2420 struct kvm_io_bus *bus;
2421
2422 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2423 for (i = 0; i < bus->dev_count; i++)
2424 if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
2425 return 0;
2426 return -EOPNOTSUPP;
2427}
2428
2429/* Caller must hold slots_lock. */
2430int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2431 struct kvm_io_device *dev)
2432{
2433 struct kvm_io_bus *new_bus, *bus;
2434
2435 bus = kvm->buses[bus_idx];
2436 if (bus->dev_count > NR_IOBUS_DEVS-1)
2437 return -ENOSPC;
2438
2439 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
2440 if (!new_bus)
2441 return -ENOMEM;
2442 memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2443 new_bus->devs[new_bus->dev_count++] = dev;
2444 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2445 synchronize_srcu_expedited(&kvm->srcu);
2446 kfree(bus);
2447
2448 return 0;
2449}
2450
2451/* Caller must hold slots_lock. */
2452int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2453 struct kvm_io_device *dev)
2454{
2455 int i, r;
2456 struct kvm_io_bus *new_bus, *bus;
2457
2458 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
2459 if (!new_bus)
2460 return -ENOMEM;
2461
2462 bus = kvm->buses[bus_idx];
2463 memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2464
2465 r = -ENOENT;
2466 for (i = 0; i < new_bus->dev_count; i++)
2467 if (new_bus->devs[i] == dev) {
2468 r = 0;
2469 new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2470 break;
2471 }
2472
2473 if (r) {
2474 kfree(new_bus);
2475 return r;
2476 }
2477
2478 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2479 synchronize_srcu_expedited(&kvm->srcu);
2480 kfree(bus);
2481 return r;
2482}
2483
2484static struct notifier_block kvm_cpu_notifier = {
2485 .notifier_call = kvm_cpu_hotplug,
2486};
2487
2488static int vm_stat_get(void *_offset, u64 *val)
2489{
2490 unsigned offset = (long)_offset;
2491 struct kvm *kvm;
2492
2493 *val = 0;
2494 raw_spin_lock(&kvm_lock);
2495 list_for_each_entry(kvm, &vm_list, vm_list)
2496 *val += *(u32 *)((void *)kvm + offset);
2497 raw_spin_unlock(&kvm_lock);
2498 return 0;
2499}
2500
2501DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2502
2503static int vcpu_stat_get(void *_offset, u64 *val)
2504{
2505 unsigned offset = (long)_offset;
2506 struct kvm *kvm;
2507 struct kvm_vcpu *vcpu;
2508 int i;
2509
2510 *val = 0;
2511 raw_spin_lock(&kvm_lock);
2512 list_for_each_entry(kvm, &vm_list, vm_list)
2513 kvm_for_each_vcpu(i, vcpu, kvm)
2514 *val += *(u32 *)((void *)vcpu + offset);
2515
2516 raw_spin_unlock(&kvm_lock);
2517 return 0;
2518}
2519
2520DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2521
2522static const struct file_operations *stat_fops[] = {
2523 [KVM_STAT_VCPU] = &vcpu_stat_fops,
2524 [KVM_STAT_VM] = &vm_stat_fops,
2525};
2526
2527static void kvm_init_debug(void)
2528{
2529 struct kvm_stats_debugfs_item *p;
2530
2531 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2532 for (p = debugfs_entries; p->name; ++p)
2533 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
2534 (void *)(long)p->offset,
2535 stat_fops[p->kind]);
2536}
2537
2538static void kvm_exit_debug(void)
2539{
2540 struct kvm_stats_debugfs_item *p;
2541
2542 for (p = debugfs_entries; p->name; ++p)
2543 debugfs_remove(p->dentry);
2544 debugfs_remove(kvm_debugfs_dir);
2545}
2546
2547static int kvm_suspend(void)
2548{
2549 if (kvm_usage_count)
2550 hardware_disable_nolock(NULL);
2551 return 0;
2552}
2553
2554static void kvm_resume(void)
2555{
2556 if (kvm_usage_count) {
2557 WARN_ON(raw_spin_is_locked(&kvm_lock));
2558 hardware_enable_nolock(NULL);
2559 }
2560}
2561
2562static struct syscore_ops kvm_syscore_ops = {
2563 .suspend = kvm_suspend,
2564 .resume = kvm_resume,
2565};
2566
2567struct page *bad_page;
2568pfn_t bad_pfn;
2569
2570static inline
2571struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2572{
2573 return container_of(pn, struct kvm_vcpu, preempt_notifier);
2574}
2575
2576static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2577{
2578 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2579
2580 kvm_arch_vcpu_load(vcpu, cpu);
2581}
2582
2583static void kvm_sched_out(struct preempt_notifier *pn,
2584 struct task_struct *next)
2585{
2586 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2587
2588 kvm_arch_vcpu_put(vcpu);
2589}
2590
2591int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2592 struct module *module)
2593{
2594 int r;
2595 int cpu;
2596
2597 r = kvm_arch_init(opaque);
2598 if (r)
2599 goto out_fail;
2600
2601 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2602
2603 if (bad_page == NULL) {
2604 r = -ENOMEM;
2605 goto out;
2606 }
2607
2608 bad_pfn = page_to_pfn(bad_page);
2609
2610 hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2611
2612 if (hwpoison_page == NULL) {
2613 r = -ENOMEM;
2614 goto out_free_0;
2615 }
2616
2617 hwpoison_pfn = page_to_pfn(hwpoison_page);
2618
2619 fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2620
2621 if (fault_page == NULL) {
2622 r = -ENOMEM;
2623 goto out_free_0;
2624 }
2625
2626 fault_pfn = page_to_pfn(fault_page);
2627
2628 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2629 r = -ENOMEM;
2630 goto out_free_0;
2631 }
2632
2633 r = kvm_arch_hardware_setup();
2634 if (r < 0)
2635 goto out_free_0a;
2636
2637 for_each_online_cpu(cpu) {
2638 smp_call_function_single(cpu,
2639 kvm_arch_check_processor_compat,
2640 &r, 1);
2641 if (r < 0)
2642 goto out_free_1;
2643 }
2644
2645 r = register_cpu_notifier(&kvm_cpu_notifier);
2646 if (r)
2647 goto out_free_2;
2648 register_reboot_notifier(&kvm_reboot_notifier);
2649
2650 /* A kmem cache lets us meet the alignment requirements of fx_save. */
2651 if (!vcpu_align)
2652 vcpu_align = __alignof__(struct kvm_vcpu);
2653 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
2654 0, NULL);
2655 if (!kvm_vcpu_cache) {
2656 r = -ENOMEM;
2657 goto out_free_3;
2658 }
2659
2660 r = kvm_async_pf_init();
2661 if (r)
2662 goto out_free;
2663
2664 kvm_chardev_ops.owner = module;
2665 kvm_vm_fops.owner = module;
2666 kvm_vcpu_fops.owner = module;
2667
2668 r = misc_register(&kvm_dev);
2669 if (r) {
2670 printk(KERN_ERR "kvm: misc device register failed\n");
2671 goto out_unreg;
2672 }
2673
2674 register_syscore_ops(&kvm_syscore_ops);
2675
2676 kvm_preempt_ops.sched_in = kvm_sched_in;
2677 kvm_preempt_ops.sched_out = kvm_sched_out;
2678
2679 kvm_init_debug();
2680
2681 return 0;
2682
2683out_unreg:
2684 kvm_async_pf_deinit();
2685out_free:
2686 kmem_cache_destroy(kvm_vcpu_cache);
2687out_free_3:
2688 unregister_reboot_notifier(&kvm_reboot_notifier);
2689 unregister_cpu_notifier(&kvm_cpu_notifier);
2690out_free_2:
2691out_free_1:
2692 kvm_arch_hardware_unsetup();
2693out_free_0a:
2694 free_cpumask_var(cpus_hardware_enabled);
2695out_free_0:
2696 if (fault_page)
2697 __free_page(fault_page);
2698 if (hwpoison_page)
2699 __free_page(hwpoison_page);
2700 __free_page(bad_page);
2701out:
2702 kvm_arch_exit();
2703out_fail:
2704 return r;
2705}
2706EXPORT_SYMBOL_GPL(kvm_init);
2707
2708void kvm_exit(void)
2709{
2710 kvm_exit_debug();
2711 misc_deregister(&kvm_dev);
2712 kmem_cache_destroy(kvm_vcpu_cache);
2713 kvm_async_pf_deinit();
2714 unregister_syscore_ops(&kvm_syscore_ops);
2715 unregister_reboot_notifier(&kvm_reboot_notifier);
2716 unregister_cpu_notifier(&kvm_cpu_notifier);
2717 on_each_cpu(hardware_disable_nolock, NULL, 1);
2718 kvm_arch_hardware_unsetup();
2719 kvm_arch_exit();
2720 free_cpumask_var(cpus_hardware_enabled);
2721 __free_page(hwpoison_page);
2722 __free_page(bad_page);
2723}
2724EXPORT_SYMBOL_GPL(kvm_exit);